Introduce XPRESS compresssion on Windows. (#1081)
Comparable with Snappy on comp ratio. Implemented using Windows API, does not require external package. Avaiable since Windows 8 and server 2012. Use -DXPRESS=1 with CMake to enable.main
Dmitri Smirnov
9 years ago
committed by
Siying Dong
parent
874c96ac1d
commit
ee221d2de0
@ -0,0 +1,270 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#include "port/win/xpress_win.h" |
||||||
|
#include <Windows.h> |
||||||
|
|
||||||
|
#include <cassert> |
||||||
|
#include <memory> |
||||||
|
#include <limits> |
||||||
|
#include <iostream> |
||||||
|
|
||||||
|
#ifdef XPRESS |
||||||
|
|
||||||
|
#ifdef JEMALLOC |
||||||
|
#include <jemalloc/jemalloc.h> |
||||||
|
#endif |
||||||
|
|
||||||
|
// Put this under ifdef so windows systems w/o this
|
||||||
|
// can still build
|
||||||
|
#include <compressapi.h> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace port { |
||||||
|
namespace xpress { |
||||||
|
|
||||||
|
// Helpers
|
||||||
|
namespace { |
||||||
|
|
||||||
|
auto CloseCompressorFun = [](void* h) { |
||||||
|
if (NULL != h) { |
||||||
|
::CloseCompressor(reinterpret_cast<COMPRESSOR_HANDLE>(h)); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
auto CloseDecompressorFun = [](void* h) { |
||||||
|
if (NULL != h) { |
||||||
|
::CloseDecompressor(reinterpret_cast<DECOMPRESSOR_HANDLE>(h)); |
||||||
|
} |
||||||
|
}; |
||||||
|
|
||||||
|
|
||||||
|
#ifdef JEMALLOC |
||||||
|
// Make sure compressors use our jemalloc if redirected
|
||||||
|
PVOID CompressorAlloc(PVOID, SIZE_T size) { |
||||||
|
return je_malloc(size); |
||||||
|
} |
||||||
|
|
||||||
|
VOID CompressorFree(PVOID, PVOID p) { |
||||||
|
if (p != NULL) { |
||||||
|
je_free(p); |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
#endif |
||||||
|
|
||||||
|
} |
||||||
|
|
||||||
|
bool Compress(const char* input, size_t length, std::string* output) { |
||||||
|
|
||||||
|
assert(input != nullptr); |
||||||
|
assert(output != nullptr); |
||||||
|
|
||||||
|
if (length == 0) { |
||||||
|
output->clear(); |
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
COMPRESS_ALLOCATION_ROUTINES* allocRoutinesPtr = nullptr; |
||||||
|
|
||||||
|
#ifdef JEMALLOC |
||||||
|
COMPRESS_ALLOCATION_ROUTINES allocationRoutines; |
||||||
|
|
||||||
|
// Init. allocation routines
|
||||||
|
allocationRoutines.Allocate = CompressorAlloc; |
||||||
|
allocationRoutines.Free = CompressorFree; |
||||||
|
allocationRoutines.UserContext = NULL; |
||||||
|
|
||||||
|
allocRoutinesPtr = &allocationRoutines; |
||||||
|
#endif |
||||||
|
|
||||||
|
COMPRESSOR_HANDLE compressor = NULL; |
||||||
|
|
||||||
|
BOOL success = CreateCompressor( |
||||||
|
COMPRESS_ALGORITHM_XPRESS, // Compression Algorithm
|
||||||
|
allocRoutinesPtr, // Optional allocation routine
|
||||||
|
&compressor); // Handle
|
||||||
|
|
||||||
|
if (!success) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr << "XPRESS: Failed to create Compressor LastError: " << |
||||||
|
GetLastError() << std::endl; |
||||||
|
#endif |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<void, decltype(CloseCompressorFun)> |
||||||
|
compressorGuard(compressor, CloseCompressorFun); |
||||||
|
|
||||||
|
SIZE_T compressedBufferSize = 0; |
||||||
|
|
||||||
|
// Query compressed buffer size.
|
||||||
|
success = ::Compress( |
||||||
|
compressor, // Compressor Handle
|
||||||
|
const_cast<char*>(input), // Input buffer
|
||||||
|
length, // Uncompressed data size
|
||||||
|
NULL, // Compressed Buffer
|
||||||
|
0, // Compressed Buffer size
|
||||||
|
&compressedBufferSize); // Compressed Data size
|
||||||
|
|
||||||
|
if (!success) { |
||||||
|
|
||||||
|
auto lastError = GetLastError(); |
||||||
|
|
||||||
|
if (lastError != ERROR_INSUFFICIENT_BUFFER) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr << |
||||||
|
"XPRESS: Failed to estimate compressed buffer size LastError " << |
||||||
|
lastError << std::endl; |
||||||
|
#endif |
||||||
|
return false; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
assert(compressedBufferSize > 0); |
||||||
|
|
||||||
|
std::string result; |
||||||
|
result.resize(compressedBufferSize); |
||||||
|
|
||||||
|
SIZE_T compressedDataSize = 0; |
||||||
|
|
||||||
|
// Compress
|
||||||
|
success = ::Compress( |
||||||
|
compressor, // Compressor Handle
|
||||||
|
const_cast<char*>(input), // Input buffer
|
||||||
|
length, // Uncompressed data size
|
||||||
|
&result[0], // Compressed Buffer
|
||||||
|
compressedBufferSize, // Compressed Buffer size
|
||||||
|
&compressedDataSize); // Compressed Data size
|
||||||
|
|
||||||
|
if (!success) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr << "XPRESS: Failed to compress LastError " << |
||||||
|
GetLastError() << std::endl; |
||||||
|
#endif |
||||||
|
return false; |
||||||
|
} |
||||||
|
|
||||||
|
result.resize(compressedDataSize); |
||||||
|
output->swap(result); |
||||||
|
|
||||||
|
return true; |
||||||
|
} |
||||||
|
|
||||||
|
char* Decompress(const char* input_data, size_t input_length, |
||||||
|
int* decompress_size) { |
||||||
|
|
||||||
|
assert(input_data != nullptr); |
||||||
|
assert(decompress_size != nullptr); |
||||||
|
|
||||||
|
if (input_length == 0) { |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
COMPRESS_ALLOCATION_ROUTINES* allocRoutinesPtr = nullptr; |
||||||
|
|
||||||
|
#ifdef JEMALLOC |
||||||
|
COMPRESS_ALLOCATION_ROUTINES allocationRoutines; |
||||||
|
|
||||||
|
// Init. allocation routines
|
||||||
|
allocationRoutines.Allocate = CompressorAlloc; |
||||||
|
allocationRoutines.Free = CompressorFree; |
||||||
|
allocationRoutines.UserContext = NULL; |
||||||
|
allocRoutinesPtr = &allocationRoutines; |
||||||
|
#endif |
||||||
|
|
||||||
|
DECOMPRESSOR_HANDLE decompressor = NULL; |
||||||
|
|
||||||
|
BOOL success = CreateDecompressor( |
||||||
|
COMPRESS_ALGORITHM_XPRESS, // Compression Algorithm
|
||||||
|
allocRoutinesPtr, // Optional allocation routine
|
||||||
|
&decompressor); // Handle
|
||||||
|
|
||||||
|
|
||||||
|
if (!success) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr <<
|
||||||
|
"XPRESS: Failed to create Decompressor LastError " << |
||||||
|
GetLastError() << std::endl; |
||||||
|
#endif |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
std::unique_ptr<void, decltype(CloseDecompressorFun)> |
||||||
|
compressorGuard(decompressor, CloseDecompressorFun); |
||||||
|
|
||||||
|
SIZE_T decompressedBufferSize = 0; |
||||||
|
|
||||||
|
success = ::Decompress( |
||||||
|
decompressor, // Compressor Handle
|
||||||
|
const_cast<char*>(input_data), // Compressed data
|
||||||
|
input_length, // Compressed data size
|
||||||
|
NULL, // Buffer set to NULL
|
||||||
|
0, // Buffer size set to 0
|
||||||
|
&decompressedBufferSize); // Decompressed Data size
|
||||||
|
|
||||||
|
if (!success) { |
||||||
|
|
||||||
|
auto lastError = GetLastError(); |
||||||
|
|
||||||
|
if (lastError != ERROR_INSUFFICIENT_BUFFER) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr <<
|
||||||
|
"XPRESS: Failed to estimate decompressed buffer size LastError " << |
||||||
|
lastError << std::endl; |
||||||
|
#endif |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
assert(decompressedBufferSize > 0); |
||||||
|
|
||||||
|
// On Windows we are limited to a 32-bit int for the
|
||||||
|
// output data size argument
|
||||||
|
// so we hopefully never get here
|
||||||
|
if (decompressedBufferSize > std::numeric_limits<int>::max()) { |
||||||
|
assert(false); |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
// The callers are deallocating using delete[]
|
||||||
|
// thus we must allocate with new[]
|
||||||
|
std::unique_ptr<char[]> outputBuffer(new char[decompressedBufferSize]); |
||||||
|
|
||||||
|
SIZE_T decompressedDataSize = 0; |
||||||
|
|
||||||
|
success = ::Decompress( |
||||||
|
decompressor, |
||||||
|
const_cast<char*>(input_data), |
||||||
|
input_length, |
||||||
|
outputBuffer.get(), |
||||||
|
decompressedBufferSize, |
||||||
|
&decompressedDataSize); |
||||||
|
|
||||||
|
if (!success) { |
||||||
|
#ifdef _DEBUG |
||||||
|
std::cerr << |
||||||
|
"XPRESS: Failed to decompress LastError " << |
||||||
|
GetLastError() << std::endl; |
||||||
|
#endif |
||||||
|
return nullptr; |
||||||
|
} |
||||||
|
|
||||||
|
*decompress_size = static_cast<int>(decompressedDataSize); |
||||||
|
|
||||||
|
// Return the raw buffer to the caller supporting the tradition
|
||||||
|
return outputBuffer.release(); |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
#endif |
||||||
|
|
||||||
|
|
||||||
@ -0,0 +1,26 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
#include <string> |
||||||
|
|
||||||
|
namespace rocksdb { |
||||||
|
namespace port { |
||||||
|
namespace xpress { |
||||||
|
|
||||||
|
bool Compress(const char* input, size_t length, std::string* output); |
||||||
|
|
||||||
|
char* Decompress(const char* input_data, size_t input_length, |
||||||
|
int* decompress_size); |
||||||
|
|
||||||
|
} |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
@ -0,0 +1,17 @@ |
|||||||
|
// Copyright (c) 2011-present, Facebook, Inc. All rights reserved.
|
||||||
|
// This source code is licensed under the BSD-style license found in the
|
||||||
|
// LICENSE file in the root directory of this source tree. An additional grant
|
||||||
|
// of patent rights can be found in the PATENTS file in the same directory.
|
||||||
|
//
|
||||||
|
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||||
|
// Use of this source code is governed by a BSD-style license that can be
|
||||||
|
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||||
|
|
||||||
|
#pragma once |
||||||
|
|
||||||
|
// Xpress on Windows is implemeted using Win API
|
||||||
|
#if defined(ROCKSDB_PLATFORM_POSIX) |
||||||
|
#error "Xpress compression not implemented" |
||||||
|
#elif defined(OS_WIN) |
||||||
|
#include "port/win/xpress_win.h" |
||||||
|
#endif |
||||||
Loading…
Reference in new issue