diff --git a/fbcode.sh b/fbcode.sh index aa6dee4fd..fc515f9bc 100644 --- a/fbcode.sh +++ b/fbcode.sh @@ -18,8 +18,12 @@ if test "$USE_HDFS"; then HDFSLIB+=" -ldl -lverify -ljava -ljvm " fi +# location of snappy headers and libraries +SNAPPY_INCLUDE=" -I ./snappy" +SNAPPY_LIBS=" -L./snappy/libs" + CC="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/gcc" -CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/g++ $JINCLUDE" +CXX="$TOOLCHAIN_EXECUTABLES/gcc/gcc-4.6.2-glibc-2.13/bin/g++ $JINCLUDE $SNAPPY_INCLUDE" AR=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ar RANLIB=$TOOLCHAIN_EXECUTABLES/binutils/binutils-2.21.1/da39a3e/bin/ranlib @@ -28,6 +32,6 @@ CFLAGS+=" -I $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-2.2.5/96de4f9/include -DHAVE_ EXEC_LDFLAGS=" -Wl,--whole-archive $TOOLCHAIN_LIB_BASE/jemalloc/jemalloc-2.2.4/96de4f9/lib/libjemalloc.a " EXEC_LDFLAGS+="-Wl,--no-whole-archive $TOOLCHAIN_LIB_BASE/libunwind/libunwind-20100810/4bc2c16/lib/libunwind.a" -EXEC_LDFLAGS+=$HDFSLIB +EXEC_LDFLAGS+="$HDFSLIB $SNAPPY_LIBS" export CC CXX AR RANLIB CFLAGS EXEC_LDFLAGS diff --git a/hdfs/README b/hdfs/README index dcedb93e1..d5085ef11 100644 --- a/hdfs/README +++ b/hdfs/README @@ -16,7 +16,7 @@ If you want to compile leveldb with hdfs support, please set the following enviroment variables appropriately: USE_HDFS=1 JAVA_HOME=/usr/local/jdk-6u22-64 - LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64/ + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64/:./snappy/libs make clean all db_bench To run dbbench, diff --git a/snappy/README b/snappy/README new file mode 100644 index 000000000..649d3ea1e --- /dev/null +++ b/snappy/README @@ -0,0 +1,6 @@ +This is a pre-compiled version of snappy 1.0.5 from +http://code.google.com/p/snappy/downloads/detail?name=snappy-1.0.5.tar.gz + +This is compiled using gcc-4.6.2-glibc-2.13 on centos5.2 +and uses jemalloc. This is here so that one can compie leveldb to use +snappy easily. diff --git a/snappy/libs/libsnappy.a b/snappy/libs/libsnappy.a new file mode 100644 index 000000000..8dbd77c36 Binary files /dev/null and b/snappy/libs/libsnappy.a differ diff --git a/snappy/libs/libsnappy.la b/snappy/libs/libsnappy.la new file mode 120000 index 000000000..e51e4065d --- /dev/null +++ b/snappy/libs/libsnappy.la @@ -0,0 +1 @@ +../libsnappy.la \ No newline at end of file diff --git a/snappy/libs/libsnappy.lai b/snappy/libs/libsnappy.lai new file mode 100644 index 000000000..a5a0e7d7a --- /dev/null +++ b/snappy/libs/libsnappy.lai @@ -0,0 +1,35 @@ +# libsnappy.la - a libtool library file +# Generated by ltmain.sh - GNU libtool 1.5.22 (1.1220.2.365 2005/12/18 22:14:06) +# +# Please DO NOT delete this file! +# It is necessary for linking the library. + +# The name that we can dlopen(3). +dlname='libsnappy.so.1' + +# Names of this library. +library_names='libsnappy.so.1.1.3 libsnappy.so.1 libsnappy.so' + +# The name of the static archive. +old_library='libsnappy.a' + +# Libraries that this one depends upon. +dependency_libs=' /home/dgadling/local/hphpbuild/gcc-install/lib/../lib64/libstdc++.la' + +# Version information for libsnappy. +current=2 +age=1 +revision=3 + +# Is this an already installed library? +installed=yes + +# Should we warn about portability when linking against -modules? +shouldnotlink=no + +# Files to dlopen/dlpreopen +dlopen='' +dlpreopen='' + +# Directory that this library needs to be installed in: +libdir='/usr/local/lib' diff --git a/snappy/libs/libsnappy.so b/snappy/libs/libsnappy.so new file mode 120000 index 000000000..032dbbccf --- /dev/null +++ b/snappy/libs/libsnappy.so @@ -0,0 +1 @@ +libsnappy.so.1.1.3 \ No newline at end of file diff --git a/snappy/libs/libsnappy.so.1 b/snappy/libs/libsnappy.so.1 new file mode 120000 index 000000000..032dbbccf --- /dev/null +++ b/snappy/libs/libsnappy.so.1 @@ -0,0 +1 @@ +libsnappy.so.1.1.3 \ No newline at end of file diff --git a/snappy/libs/libsnappy.so.1.1.3 b/snappy/libs/libsnappy.so.1.1.3 new file mode 100755 index 000000000..3d8805457 Binary files /dev/null and b/snappy/libs/libsnappy.so.1.1.3 differ diff --git a/snappy/libs/lt-snappy_unittest b/snappy/libs/lt-snappy_unittest new file mode 100755 index 000000000..209abd1af Binary files /dev/null and b/snappy/libs/lt-snappy_unittest differ diff --git a/snappy/libs/snappy_unittest b/snappy/libs/snappy_unittest new file mode 100755 index 000000000..91d6d3509 Binary files /dev/null and b/snappy/libs/snappy_unittest differ diff --git a/snappy/snappy-stubs-public.h b/snappy/snappy-stubs-public.h new file mode 100644 index 000000000..9ee4ca52b --- /dev/null +++ b/snappy/snappy-stubs-public.h @@ -0,0 +1,85 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: sesse@google.com (Steinar H. Gunderson) +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// Various type stubs for the open-source version of Snappy. +// +// This file cannot include config.h, as it is included from snappy.h, +// which is a public header. Instead, snappy-stubs-public.h is generated by +// from snappy-stubs-public.h.in at configure time. + +#ifndef UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ +#define UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ + +#if 1 +#include +#endif + +#if 1 +#include +#endif + +#define SNAPPY_MAJOR 1 +#define SNAPPY_MINOR 0 +#define SNAPPY_PATCHLEVEL 5 +#define SNAPPY_VERSION \ + ((SNAPPY_MAJOR << 16) | (SNAPPY_MINOR << 8) | SNAPPY_PATCHLEVEL) + +#include + +namespace snappy { + +#if 1 +typedef int8_t int8; +typedef uint8_t uint8; +typedef int16_t int16; +typedef uint16_t uint16; +typedef int32_t int32; +typedef uint32_t uint32; +typedef int64_t int64; +typedef uint64_t uint64; +#else +typedef signed char int8; +typedef unsigned char uint8; +typedef short int16; +typedef unsigned short uint16; +typedef int int32; +typedef unsigned int uint32; +typedef long long int64; +typedef unsigned long long uint64; +#endif + +typedef std::string string; + +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +} // namespace snappy + +#endif // UTIL_SNAPPY_OPENSOURCE_SNAPPY_STUBS_PUBLIC_H_ diff --git a/snappy/snappy.h b/snappy/snappy.h new file mode 100644 index 000000000..8c2075fef --- /dev/null +++ b/snappy/snappy.h @@ -0,0 +1,155 @@ +// Copyright 2005 and onwards Google Inc. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +// A light-weight compression algorithm. It is designed for speed of +// compression and decompression, rather than for the utmost in space +// savings. +// +// For getting better compression ratios when you are compressing data +// with long repeated sequences or compressing data that is similar to +// other data, while still compressing fast, you might look at first +// using BMDiff and then compressing the output of BMDiff with +// Snappy. + +#ifndef UTIL_SNAPPY_SNAPPY_H__ +#define UTIL_SNAPPY_SNAPPY_H__ + +#include +#include + +#include "snappy-stubs-public.h" + +namespace snappy { + class Source; + class Sink; + + // ------------------------------------------------------------------------ + // Generic compression/decompression routines. + // ------------------------------------------------------------------------ + + // Compress the bytes read from "*source" and append to "*sink". Return the + // number of bytes written. + size_t Compress(Source* source, Sink* sink); + + bool GetUncompressedLength(Source* source, uint32* result); + + // ------------------------------------------------------------------------ + // Higher-level string based routines (should be sufficient for most users) + // ------------------------------------------------------------------------ + + // Sets "*output" to the compressed version of "input[0,input_length-1]". + // Original contents of *output are lost. + // + // REQUIRES: "input[]" is not an alias of "*output". + size_t Compress(const char* input, size_t input_length, string* output); + + // Decompresses "compressed[0,compressed_length-1]" to "*uncompressed". + // Original contents of "*uncompressed" are lost. + // + // REQUIRES: "compressed[]" is not an alias of "*uncompressed". + // + // returns false if the message is corrupted and could not be decompressed + bool Uncompress(const char* compressed, size_t compressed_length, + string* uncompressed); + + + // ------------------------------------------------------------------------ + // Lower-level character array based routines. May be useful for + // efficiency reasons in certain circumstances. + // ------------------------------------------------------------------------ + + // REQUIRES: "compressed" must point to an area of memory that is at + // least "MaxCompressedLength(input_length)" bytes in length. + // + // Takes the data stored in "input[0..input_length]" and stores + // it in the array pointed to by "compressed". + // + // "*compressed_length" is set to the length of the compressed output. + // + // Example: + // char* output = new char[snappy::MaxCompressedLength(input_length)]; + // size_t output_length; + // RawCompress(input, input_length, output, &output_length); + // ... Process(output, output_length) ... + // delete [] output; + void RawCompress(const char* input, + size_t input_length, + char* compressed, + size_t* compressed_length); + + // Given data in "compressed[0..compressed_length-1]" generated by + // calling the Snappy::Compress routine, this routine + // stores the uncompressed data to + // uncompressed[0..GetUncompressedLength(compressed)-1] + // returns false if the message is corrupted and could not be decrypted + bool RawUncompress(const char* compressed, size_t compressed_length, + char* uncompressed); + + // Given data from the byte source 'compressed' generated by calling + // the Snappy::Compress routine, this routine stores the uncompressed + // data to + // uncompressed[0..GetUncompressedLength(compressed,compressed_length)-1] + // returns false if the message is corrupted and could not be decrypted + bool RawUncompress(Source* compressed, char* uncompressed); + + // Returns the maximal size of the compressed representation of + // input data that is "source_bytes" bytes in length; + size_t MaxCompressedLength(size_t source_bytes); + + // REQUIRES: "compressed[]" was produced by RawCompress() or Compress() + // Returns true and stores the length of the uncompressed data in + // *result normally. Returns false on parsing error. + // This operation takes O(1) time. + bool GetUncompressedLength(const char* compressed, size_t compressed_length, + size_t* result); + + // Returns true iff the contents of "compressed[]" can be uncompressed + // successfully. Does not return the uncompressed data. Takes + // time proportional to compressed_length, but is usually at least + // a factor of four faster than actual decompression. + bool IsValidCompressedBuffer(const char* compressed, + size_t compressed_length); + + // *** DO NOT CHANGE THE VALUE OF kBlockSize *** + // + // New Compression code chops up the input into blocks of at most + // the following size. This ensures that back-references in the + // output never cross kBlockSize block boundaries. This can be + // helpful in implementing blocked decompression. However the + // decompression code should not rely on this guarantee since older + // compression code may not obey it. + static const int kBlockLog = 15; + static const size_t kBlockSize = 1 << kBlockLog; + + static const int kMaxHashTableBits = 14; + static const size_t kMaxHashTableSize = 1 << kMaxHashTableBits; + +} // end namespace snappy + + +#endif // UTIL_SNAPPY_SNAPPY_H__