From c2fda55cfe14918ac4f67f43c885ec2749dab88d Mon Sep 17 00:00:00 2001 From: Mike Orr Date: Tue, 20 May 2014 17:22:12 -0400 Subject: [PATCH 1/8] hdfs cleanup and compile test against CDH 4.4. --- build_tools/build_detect_platform | 2 +- hdfs/README | 13 +- hdfs/env_hdfs.h | 2 +- hdfs/hdfs.h | 477 ------------------------------ hdfs/libhdfs.a | Bin 65218 -> 0 bytes hdfs/setup.sh | 7 + util/env_hdfs.cc | 48 ++- 7 files changed, 52 insertions(+), 497 deletions(-) delete mode 100644 hdfs/hdfs.h delete mode 100644 hdfs/libhdfs.a create mode 100644 hdfs/setup.sh diff --git a/build_tools/build_detect_platform b/build_tools/build_detect_platform index 88aa216ad..c8ed00487 100755 --- a/build_tools/build_detect_platform +++ b/build_tools/build_detect_platform @@ -289,7 +289,7 @@ if test "$USE_HDFS"; then exit 1 fi HDFS_CCFLAGS="$HDFS_CCFLAGS -I$JAVA_HOME/include -I$JAVA_HOME/include/linux -DUSE_HDFS" - HDFS_LDFLAGS="$HDFS_LDFLAGS -Wl,--no-whole-archive hdfs/libhdfs.a -L$JAVA_HOME/jre/lib/amd64" + HDFS_LDFLAGS="$HDFS_LDFLAGS -Wl,--no-whole-archive -lhdfs -L$JAVA_HOME/jre/lib/amd64" HDFS_LDFLAGS="$HDFS_LDFLAGS -L$JAVA_HOME/jre/lib/amd64/server -L$GLIBC_RUNTIME_PATH/lib" HDFS_LDFLAGS="$HDFS_LDFLAGS -ldl -lverify -ljava -ljvm" COMMON_FLAGS="$COMMON_FLAGS $HDFS_CCFLAGS" diff --git a/hdfs/README b/hdfs/README index 9b7d0a64d..f4f1106e4 100644 --- a/hdfs/README +++ b/hdfs/README @@ -1,19 +1,16 @@ This directory contains the hdfs extensions needed to make rocksdb store files in HDFS. -The hdfs.h file is copied from the Apache Hadoop 1.0 source code. -It defines the libhdfs library -(http://hadoop.apache.org/common/docs/r0.20.2/libhdfs.html) to access -data in HDFS. The libhdfs.a is copied from the Apache Hadoop 1.0 build. -It implements the API defined in hdfs.h. If your hadoop cluster is running -a different hadoop release, then install these two files manually from your -hadoop distribution and then recompile rocksdb. +It has been compiled and testing against CDH 4.4 (2.0.0+1475-1.cdh4.4.0.p0.23~precise-cdh4.4.0). + +The configuration assumes that packages libhdfs0, libhdfs0-dev are +installed which basically means that hdfs.h is in /usr/include and libhdfs in /usr/lib The env_hdfs.h file defines the rocksdb objects that are needed to talk to an underlying filesystem. If you want to compile rocksdb with hdfs support, please set the following -enviroment variables appropriately: +enviroment variables appropriately (also defined in setup.sh for convenience) USE_HDFS=1 JAVA_HOME=/usr/local/jdk-6u22-64 LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/jdk-6u22-64/jre/lib/amd64/server:/usr/local/jdk-6u22-64/jre/lib/amd64/:./snappy/libs diff --git a/hdfs/env_hdfs.h b/hdfs/env_hdfs.h index e6fb8db12..5e7de77d3 100644 --- a/hdfs/env_hdfs.h +++ b/hdfs/env_hdfs.h @@ -14,7 +14,7 @@ #include "rocksdb/status.h" #ifdef USE_HDFS -#include "hdfs/hdfs.h" +#include namespace rocksdb { diff --git a/hdfs/hdfs.h b/hdfs/hdfs.h deleted file mode 100644 index 8e8dfecb8..000000000 --- a/hdfs/hdfs.h +++ /dev/null @@ -1,477 +0,0 @@ -// Copyright (c) 2013, Facebook, Inc. All rights reserved. -// This source code is licensed under the BSD-style license found in the -// LICENSE file in the root directory of this source tree. An additional grant -// of patent rights can be found in the PATENTS file in the same directory. -// -#ifndef LIBHDFS_HDFS_H -#define LIBHDFS_HDFS_H - -#include -#include - -#include -#include -#include -#include -#include -#include -#include - -#include - -#ifndef O_RDONLY -#define O_RDONLY 1 -#endif - -#ifndef O_WRONLY -#define O_WRONLY 2 -#endif - -#ifndef EINTERNAL -#define EINTERNAL 255 -#endif - - -/** All APIs set errno to meaningful values */ -#ifdef __cplusplus -extern "C" { -#endif - - /** - * Some utility decls used in libhdfs. - */ - - typedef int32_t tSize; /// size of data for read/write io ops - typedef time_t tTime; /// time type in seconds - typedef int64_t tOffset;/// offset within the file - typedef uint16_t tPort; /// port - typedef enum tObjectKind { - kObjectKindFile = 'F', - kObjectKindDirectory = 'D', - } tObjectKind; - - - /** - * The C reflection of org.apache.org.hadoop.FileSystem . - */ - typedef void* hdfsFS; - - - /** - * The C equivalent of org.apache.org.hadoop.FSData(Input|Output)Stream . - */ - enum hdfsStreamType - { - UNINITIALIZED = 0, - INPUT = 1, - OUTPUT = 2, - }; - - - /** - * The 'file-handle' to a file in hdfs. - */ - struct hdfsFile_internal { - void* file; - enum hdfsStreamType type; - }; - typedef struct hdfsFile_internal* hdfsFile; - - - /** - * hdfsConnectAsUser - Connect to a hdfs file system as a specific user - * Connect to the hdfs. - * @param host A string containing either a host name, or an ip address - * of the namenode of a hdfs cluster. 'host' should be passed as NULL if - * you want to connect to local filesystem. 'host' should be passed as - * 'default' (and port as 0) to used the 'configured' filesystem - * (core-site/core-default.xml). - * @param port The port on which the server is listening. - * @param user the user name (this is hadoop domain user). Or NULL is equivelant to hhdfsConnect(host, port) - * @param groups the groups (these are hadoop domain groups) - * @return Returns a handle to the filesystem or NULL on error. - */ - hdfsFS hdfsConnectAsUser(const char* host, tPort port, const char *user , const char *groups[], int groups_size ); - - - /** - * hdfsConnect - Connect to a hdfs file system. - * Connect to the hdfs. - * @param host A string containing either a host name, or an ip address - * of the namenode of a hdfs cluster. 'host' should be passed as NULL if - * you want to connect to local filesystem. 'host' should be passed as - * 'default' (and port as 0) to used the 'configured' filesystem - * (core-site/core-default.xml). - * @param port The port on which the server is listening. - * @return Returns a handle to the filesystem or NULL on error. - */ - hdfsFS hdfsConnect(const char* host, tPort port); - - - /** - * This are the same as hdfsConnectAsUser except that every invocation returns a new FileSystem handle. - * Applications should call a hdfsDisconnect for every call to hdfsConnectAsUserNewInstance. - */ - hdfsFS hdfsConnectAsUserNewInstance(const char* host, tPort port, const char *user , const char *groups[], int groups_size ); - hdfsFS hdfsConnectNewInstance(const char* host, tPort port); - hdfsFS hdfsConnectPath(const char* uri); - - /** - * hdfsDisconnect - Disconnect from the hdfs file system. - * Disconnect from hdfs. - * @param fs The configured filesystem handle. - * @return Returns 0 on success, -1 on error. - */ - int hdfsDisconnect(hdfsFS fs); - - - /** - * hdfsOpenFile - Open a hdfs file in given mode. - * @param fs The configured filesystem handle. - * @param path The full path to the file. - * @param flags - an | of bits/fcntl.h file flags - supported flags are O_RDONLY, O_WRONLY (meaning create or overwrite i.e., implies O_TRUNCAT), - * O_WRONLY|O_APPEND. Other flags are generally ignored other than (O_RDWR || (O_EXCL & O_CREAT)) which return NULL and set errno equal ENOTSUP. - * @param bufferSize Size of buffer for read/write - pass 0 if you want - * to use the default configured values. - * @param replication Block replication - pass 0 if you want to use - * the default configured values. - * @param blocksize Size of block - pass 0 if you want to use the - * default configured values. - * @return Returns the handle to the open file or NULL on error. - */ - hdfsFile hdfsOpenFile(hdfsFS fs, const char* path, int flags, - int bufferSize, short replication, tSize blocksize); - - - /** - * hdfsCloseFile - Close an open file. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @return Returns 0 on success, -1 on error. - */ - int hdfsCloseFile(hdfsFS fs, hdfsFile file); - - - /** - * hdfsExists - Checks if a given path exsits on the filesystem - * @param fs The configured filesystem handle. - * @param path The path to look for - * @return Returns 0 on exists, 1 on non-exists, -1/-2 on error. - */ - int hdfsExists(hdfsFS fs, const char *path); - - - /** - * hdfsSeek - Seek to given offset in file. - * This works only for files opened in read-only mode. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @param desiredPos Offset into the file to seek into. - * @return Returns 0 on success, -1 on error. - */ - int hdfsSeek(hdfsFS fs, hdfsFile file, tOffset desiredPos); - - - /** - * hdfsTell - Get the current offset in the file, in bytes. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @return Current offset, -1 on error. - */ - tOffset hdfsTell(hdfsFS fs, hdfsFile file); - - - /** - * hdfsRead - Read data from an open file. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @param buffer The buffer to copy read bytes into. - * @param length The length of the buffer. - * @return Returns the number of bytes actually read, possibly less - * than than length;-1 on error. - */ - tSize hdfsRead(hdfsFS fs, hdfsFile file, void* buffer, tSize length); - - - /** - * hdfsPread - Positional read of data from an open file. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @param position Position from which to read - * @param buffer The buffer to copy read bytes into. - * @param length The length of the buffer. - * @return Returns the number of bytes actually read, possibly less than - * than length;-1 on error. - */ - tSize hdfsPread(hdfsFS fs, hdfsFile file, tOffset position, - void* buffer, tSize length); - - - /** - * hdfsWrite - Write data into an open file. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @param buffer The data. - * @param length The no. of bytes to write. - * @return Returns the number of bytes written, -1 on error. - */ - tSize hdfsWrite(hdfsFS fs, hdfsFile file, const void* buffer, - tSize length); - - - /** - * hdfsWrite - Flush the data. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @return Returns 0 on success, -1 on error. - */ - int hdfsFlush(hdfsFS fs, hdfsFile file); - - /** - * hdfsSync - Sync the data to persistent store. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @return Returns 0 on success, -1 on error. - */ - int hdfsSync(hdfsFS fs, hdfsFile file); - - /** - * hdfsGetNumReplicasInPipeline - get number of remaining replicas in - * pipeline - * @param fs The configured filesystem handle - * @param file the file handle - * @return returns the # of datanodes in the write pipeline; -1 on error - */ - int hdfsGetNumCurrentReplicas(hdfsFS, hdfsFile file); - - /** - * hdfsAvailable - Number of bytes that can be read from this - * input stream without blocking. - * @param fs The configured filesystem handle. - * @param file The file handle. - * @return Returns available bytes; -1 on error. - */ - int hdfsAvailable(hdfsFS fs, hdfsFile file); - - - /** - * hdfsCopy - Copy file from one filesystem to another. - * @param srcFS The handle to source filesystem. - * @param src The path of source file. - * @param dstFS The handle to destination filesystem. - * @param dst The path of destination file. - * @return Returns 0 on success, -1 on error. - */ - int hdfsCopy(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); - - - /** - * hdfsMove - Move file from one filesystem to another. - * @param srcFS The handle to source filesystem. - * @param src The path of source file. - * @param dstFS The handle to destination filesystem. - * @param dst The path of destination file. - * @return Returns 0 on success, -1 on error. - */ - int hdfsMove(hdfsFS srcFS, const char* src, hdfsFS dstFS, const char* dst); - - - /** - * hdfsDelete - Delete file. - * @param fs The configured filesystem handle. - * @param path The path of the file. - * @return Returns 0 on success, -1 on error. - */ - int hdfsDelete(hdfsFS fs, const char* path); - - - /** - * hdfsRename - Rename file. - * @param fs The configured filesystem handle. - * @param oldPath The path of the source file. - * @param newPath The path of the destination file. - * @return Returns 0 on success, -1 on error. - */ - int hdfsRename(hdfsFS fs, const char* oldPath, const char* newPath); - - - /** - * hdfsGetWorkingDirectory - Get the current working directory for - * the given filesystem. - * @param fs The configured filesystem handle. - * @param buffer The user-buffer to copy path of cwd into. - * @param bufferSize The length of user-buffer. - * @return Returns buffer, NULL on error. - */ - char* hdfsGetWorkingDirectory(hdfsFS fs, char *buffer, size_t bufferSize); - - - /** - * hdfsSetWorkingDirectory - Set the working directory. All relative - * paths will be resolved relative to it. - * @param fs The configured filesystem handle. - * @param path The path of the new 'cwd'. - * @return Returns 0 on success, -1 on error. - */ - int hdfsSetWorkingDirectory(hdfsFS fs, const char* path); - - - /** - * hdfsCreateDirectory - Make the given file and all non-existent - * parents into directories. - * @param fs The configured filesystem handle. - * @param path The path of the directory. - * @return Returns 0 on success, -1 on error. - */ - int hdfsCreateDirectory(hdfsFS fs, const char* path); - - - /** - * hdfsSetReplication - Set the replication of the specified - * file to the supplied value - * @param fs The configured filesystem handle. - * @param path The path of the file. - * @return Returns 0 on success, -1 on error. - */ - int hdfsSetReplication(hdfsFS fs, const char* path, int16_t replication); - - - /** - * hdfsFileInfo - Information about a file/directory. - */ - typedef struct { - tObjectKind mKind; /* file or directory */ - char *mName; /* the name of the file */ - tTime mLastMod; /* the last modification time for the file in seconds */ - tOffset mSize; /* the size of the file in bytes */ - short mReplication; /* the count of replicas */ - tOffset mBlockSize; /* the block size for the file */ - char *mOwner; /* the owner of the file */ - char *mGroup; /* the group associated with the file */ - short mPermissions; /* the permissions associated with the file */ - tTime mLastAccess; /* the last access time for the file in seconds */ - } hdfsFileInfo; - - - /** - * hdfsListDirectory - Get list of files/directories for a given - * directory-path. hdfsFreeFileInfo should be called to deallocate memory if - * the function returns non-NULL value. - * @param fs The configured filesystem handle. - * @param path The path of the directory. - * @param numEntries Set to the number of files/directories in path. - * @return Returns a dynamically-allocated array of hdfsFileInfo - * objects; NULL if empty or on error. - * on error, numEntries will be -1. - */ - hdfsFileInfo *hdfsListDirectory(hdfsFS fs, const char* path, - int *numEntries); - - - /** - * hdfsGetPathInfo - Get information about a path as a (dynamically - * allocated) single hdfsFileInfo struct. hdfsFreeFileInfo should be - * called when the pointer is no longer needed. - * @param fs The configured filesystem handle. - * @param path The path of the file. - * @return Returns a dynamically-allocated hdfsFileInfo object; - * NULL on error. - */ - hdfsFileInfo *hdfsGetPathInfo(hdfsFS fs, const char* path); - - - /** - * hdfsFreeFileInfo - Free up the hdfsFileInfo array (including fields) - * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo - * objects. - * @param numEntries The size of the array. - */ - void hdfsFreeFileInfo(hdfsFileInfo *hdfsFileInfo, int numEntries); - - - /** - * hdfsGetHosts - Get hostnames where a particular block (determined by - * pos & blocksize) of a file is stored. The last element in the array - * is NULL. Due to replication, a single block could be present on - * multiple hosts. - * @param fs The configured filesystem handle. - * @param path The path of the file. - * @param start The start of the block. - * @param length The length of the block. - * @return Returns a dynamically-allocated 2-d array of blocks-hosts; - * NULL on error. - */ - char*** hdfsGetHosts(hdfsFS fs, const char* path, - tOffset start, tOffset length); - - - /** - * hdfsFreeHosts - Free up the structure returned by hdfsGetHosts - * @param hdfsFileInfo The array of dynamically-allocated hdfsFileInfo - * objects. - * @param numEntries The size of the array. - */ - void hdfsFreeHosts(char ***blockHosts); - - - /** - * hdfsGetDefaultBlockSize - Get the optimum blocksize. - * @param fs The configured filesystem handle. - * @return Returns the blocksize; -1 on error. - */ - tOffset hdfsGetDefaultBlockSize(hdfsFS fs); - - - /** - * hdfsGetCapacity - Return the raw capacity of the filesystem. - * @param fs The configured filesystem handle. - * @return Returns the raw-capacity; -1 on error. - */ - tOffset hdfsGetCapacity(hdfsFS fs); - - - /** - * hdfsGetUsed - Return the total raw size of all files in the filesystem. - * @param fs The configured filesystem handle. - * @return Returns the total-size; -1 on error. - */ - tOffset hdfsGetUsed(hdfsFS fs); - - /** - * hdfsChown - * @param fs The configured filesystem handle. - * @param path the path to the file or directory - * @param owner this is a string in Hadoop land. Set to null or "" if only setting group - * @param group this is a string in Hadoop land. Set to null or "" if only setting user - * @return 0 on success else -1 - */ - int hdfsChown(hdfsFS fs, const char* path, const char *owner, const char *group); - - /** - * hdfsChmod - * @param fs The configured filesystem handle. - * @param path the path to the file or directory - * @param mode the bitmask to set it to - * @return 0 on success else -1 - */ - int hdfsChmod(hdfsFS fs, const char* path, short mode); - - /** - * hdfsUtime - * @param fs The configured filesystem handle. - * @param path the path to the file or directory - * @param mtime new modification time or 0 for only set access time in seconds - * @param atime new access time or 0 for only set modification time in seconds - * @return 0 on success else -1 - */ - int hdfsUtime(hdfsFS fs, const char* path, tTime mtime, tTime atime); - -#ifdef __cplusplus -} -#endif - -#endif /*LIBHDFS_HDFS_H*/ - -/** - * vim: ts=4: sw=4: et - */ diff --git a/hdfs/libhdfs.a b/hdfs/libhdfs.a deleted file mode 100644 index 4d1f19f0b9cceb4a8923d7c876c64841eceb2495..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 65218 zcmd_T3w&HvwLgCP(3V0;0l3 zB?L1Lk?N)1UO^3tzZ(>>DvyXQgvV{-lYkY&wQ5x4PE3E7qA1Z@?f<*hUTbE}&P-a0 zcz^%j?|j;vbH01+wbxpE?X@50%-Qd{u)LvS!-7e(LfOZhbIzGJZ|=Mc&Yv9$;U5nB zHx#y5s>eOtd<*^Ovd#A=F@v16ufhEz#SVP^7 z2Ek>zbX=#ajxMgQWMeUFu1W=}Yp<`nF1j)r+W@7nt*N`Fyk>Dtd1K@1C<=;_V{PQL zWtB@C>Navx05?L}@roE+Y*9l)`HgF04XBSSeAP7{M&+!kk{rbHGF0KJDwhyk8;vc$ zY+1PWdKRg;F4OL%#m}2JFBF<{{=B*PJDb`#{+@sCxuMW`XP!U>7^ z{zyyH{vbA$Xd=lVl5Fk*he#3$$)@3hed!Fdks%dH#0Nl`a+LGl!jwRoQV07Yi7@ie z|EAQkMEAEzkVomyWAul(1}XXjx?vPL9K>fuk{2VH*GwBgx;qV-P{6s`M4K~d>F1x1lh6%;Mp z5=m^E2)druCHO1+B6``vVg&`at|TFoPP!z6VwYB;jkN6IEb#)zhWHGAkF&V+MCg_(Y<&;IZ!3Zsk6ixYuNH zAE1mqt^GI&_dZ`&>I$W6B@5r4Jdk@&0BN*8hC$}~96bVTH9B3t$ilR$}G)K1{f8~D_|9;YRV z)~j%;N<0)JuqV77l+-*s1x8DSWRjzLhLZ zRD&OSE3zh=wr8~SA_Z7Bde-JOVDu|aAj+7`Z#t9@wGL9wynI?ELO>it zg+hn{2r&@EZ&q7SILKwIit#vG>)K!(J)w0yBY8~LuK8&Wei83YguGOr=b z*m2ExGIF*Zo3^tzuoY90_Odo8vz2m#a3GRcGChO~^jb@%PsK$CT}+AeESWx$ZQC<` zI-M7>&ZZG%G;i&W-6ST+tJfCDY4o~#*ywc!-FSL^%n=N*lC|yz9V#>~ryHYj`8akf zSA|O)rL)ak%i~PT<5aq7;no|(H+d#GA_JW68-P1?XQNh>mKZm zg!?T~s?v30tW?hvkb`uajT74es_deh-#-vZ^hRFmi?pQ1MA|n^5zB?U3m$3bn3>$h z9Fk4leiZW@JKjwm!G7K4RbtnXc=2eJ*!?HgcMDaItHd2%k?c3rf_5B9+3y*}ftGOD zLlf0|(&3@(kK_{ZE^ZhRce8_U=F?3}Q#V31?zoX<)Z`WBMTK?lYxJ`e^>InY%_w2zgdf!GjUPsVCn34H{|CtW%BYyMr zhX5Ur({ym(dBy1U0Nr?ceby1U4z8VUjFM|@pH}ZEQA_CkDr~mP>E<8SjXj3a{;%uC z2KTd4|Ic(|e}0+u{U7PZnz-y7-PkApiyMpU#k-V`!I&vkGa|aZZR2Z>dR2moEFp|J~;SsN~Oh~ z_A+YcsXT*fS_i_&$zKA(u=-3Bb-*E>GNZ~_P%SY|iW+(+%1OCi-A(p+PJ;y}5P)@% z4F4d~@^r-X+*4_=mf%?(R)J_vmDolTtVH`d{9TghUXp01uD>eLx)vujP8-y18!;0N zfr5$|Trh!DZgmsl@YW4LYZ{eI17ZirbRu$)kr}cOmo|h{#;osBzw4<8O@-> z?I}9ZI&UOL%{B&fb0u4Vz5}a@zaPELdbvlUa11{5+`qB`kVNBW5v?IV$i?PmgG;l~S zAn;tm<;HNl-HoB!w-Mo&rt(yOa1tF5YNc8zzt@a zA#zu;l{_gy1ABI%szi9Vj3c)r9fQqqo1O1BwPoh17!c%7L|2$UaiC=v6&n<^CSVN= zZ7?->XcJkuy{UOcvaAg=+o{16N(!SL=b!7+JfE6nB=P%5;%_SwKg}{z(dcIS6zqb| z@;B(nli_CAAni4`b{E`w8Te-TWI^kEK$yKzQ-kQA$S*pJgyq~fP4wpnS@c3DI#pT= zujxUo&*T>iDdS<$fg=~|+z}QHcSMpawX3V;N9Vu#}qtXw!7frl`o?XAQi`h742nc zMjt}UrEYb}^x28E(+izxLx~lCm+hx4QC#NKnDjYItxY1@R;}{WbOtOFk8MtYB$*K z#~J_2u-M4UlmjFBA6Z5v3nRRG%RK0!i!%<}1Y@oQxA=gG0B6HyH27h`h6-U+u?j^X zUP6{pPzp%{GzxlvZ8A=^)}K21 zMbr^3`Syf+i8WMq7Ud=2V~~e!_C-!&OZ1`x>O%(vecnizoNG*jijDUhvz!8<60~{@ zBS6OUe}hLiPLQE#X*2}1;Nfr|g3z7ac*vIs-=7TM54&ti+hxnX?Id@i2~LRxdklIK zEALbbVd2gWc%6lAWo;jDZr9=5uEV)q8~aC_7^w|L>+cEg<6HGcdNL6s@9W6reUA5P z;_WEleX+B3?Dt-9$~~H-2M#@9m-_^lyU#gCGe#-N_?<9PAFm$5T&R|IETpwJb)$+v z_C~764%^Y-<9()zHvQUp?mjZu&_)WJ;e9Pl`+}R!M4=?#ie%eDUdIY=Pp;f2)z+sy z;G%=i4xL5xvA5#y!KQaa+~eV7**>osiIKV5ddS=`A3H{0>eOEyJeUmcON4JthHpir zq1rZ;PZ8Nxek8GS>ykwHcFzM&7S`+mt(!?99D~o~a)M@df@UzLN^CwHCs6hrf->yu zv;E+_!}Nm;gIPIT;r9HlkPP3BUak`Yv&bt8ePqT877F0>@00Hd^3txt{|Su`amvNhcj*1%zyN zA!H}~2Hgt;-Vebk39Yx;wchUJN+k0uA_J!NX7!8IPr{kQkQW6qTjHyHR4S>w!@x9$`=f zKS9EW8Db254A$9?KnzP_xr9d(M`#_}z`!kQ>EzYD&N^=NFd=Jjkg>^@eQl(pbb@LU z>HVgP@D%$n1ZZR5eyj8C?Cx(X`}S7Q!EN|K84WSA`#a@Erbu(+D4g!_c(}ElxG+Dy z0zZTx9)n<>fr{4tlwHy5(fkLx*+Z}#Gu-!z%H&^uFZRu!kp)%Mp((DaXax%Y6ExPw z|1D3G0gaEX({Q22%;{h3vxD5)eE;fZ-Mh&OQes~J8teHNa~-NfHu%?@upHO%AEw0( z!KZeTlxS_VZUGEbS{yC9O;p|_@^}N=i-NsPoqUZd9-hYiOT(Xz@Q+nwz&FP<+(?e7=s&+?eo1>IYh~{*?jIRK0vR#F2J>k7j176;= zBlE1%8%1!#;R)+19Y!DGwJD0W(9YAK}K7{tFa~#Utv;xSJ|?xa?&#vhA2$ zbzpJ@mvNJ;d007`jT;P~Fu=wI8i{)ush2Kjgbk<6-~XGJk#eum+7Z%r*lLHFhwX>V zbs+5Lfack}A-ICN+w$?Hii#uVQepq6F%E8&?bZ?{fr4~SzOe}=;T;9o? z7l8}wM5&V@%dD%Rmtq%z3DgV@Zk4O<`@|$nCyFbj~#@p9T@r%YS{T zxsXiE-IqEuBj(V&wD2cbk_os$jdb7IO*L{SS&C}pPO@V%d}jiWb1{DzF{&rug?ZgN zq1K3mB%RpEG>KRVCX)~wd$ zMuF&p<6T;bIoHA;IdAa_9G_k=a}TOwsKaEMHC_XA<5ro z5rnk1tKpNOE=B5nBr($VvsCYbR6Ou|&^-;X zb~grnsPfqrT<bu(~e_$vw@{0cIRUI}U2A8sRgeYPhdt?iwK_wF{fe+D;f z?hdS0X7wrQ=ZLSjeOQyH?X{f>P5;#A44tutU?Q)@!1dYOzhrYmsOs(%US|<+-=4Ys z2Ris-mG|u<ue)4xum_n`1@Un!rqMflWC@1d#Fw;}k9 ze3@?^C?I1n_%nXjn&0Ar>9$6ajupMZa0~P1i z1d8p487OkhJPz@K+S?%+2XzW#ZK<=LhM-2e3GV&zW(ePbu+~W;BYlJDj1GvTrQ1mR zu6ms4m9g~ed%Qd$zc4+FpG%ECg}N+L?d~^JQ$gXBh$>Zc%*o3ON{uA&>G)g1Ef zB>#FBitT`bAv2KXbv!iWNg`f%p_lT)T`k>XB3O*M@ZU*yb%jpUBwm|wzs7N|%?#c* zmR7L4K>WnOxX3t6$J}$|mZq)@B{3Pi8I-c*EvLctfHHDyfE>v(dL9Ph+?zKTtJA={ z-Tiv9J^jk#>lyOnc~a|osw<58uN>W+2cOI`=gGKDn{%?eVbM|n-5A^cgu}lpChM(5 zv2CwY2t6^a_B&hQ2Y%x|)QP37T(%yUp(C53o*@15p74ugN;wU^BN=|t7(VGH7%kx! z#~@*Tc<1I{qS#JWng2+Y{Ru2Q_ETtu&56}MeymhsPk4_jjZcH0!Hr*8dte^X4L+61 zB+CQ^NRxXrl*bYMBa7Xm#ZDUw+J32@aT!iBaY0PuMFDytORiEM?LK(0CAb)Q326ClYtg4|f&Zekb}$B)S&A5Q(_yAn{V?xEoiqr}GWJX+<#(zQd=i zA2mf<_JuH^MaMwnreLVPCwzb}is-lh+7h5U%z8legIUuRbO2I~O32myO^-;cci7KO z)CH|Ku((YrHlhCd%DocSC1%Cgi+*AfpDOMkQ|=wed2o;>yD+<(tYGznKt#V@(hN8~ z2f?}!zcY?ilAJ&N)VI{TC=Zu`QJZ~62w%yXetq(BZg4$e7J;7;(s+{`2}2bQQadnO zuy13nqh&DFN5)P$^`WQoq7U&K;9$3%zWV?n1|Cm~h&!Y=VT|ga%|8I4S=BPg&#~7|k0-{o^ch&91|=$d-Mb zRQ2A&8U2+#y;;9Qk<(!w{Z?6du{@8faK$__V@d*-z@zj4nMzI@lyFo63#%4>s>C;l>leti7I z@Wqj$C&fmP^G&OtPdw_`Xv`auGoo%4vu!0iSU%TSJE84m-1j@ zz61TbKE3RMVo1;ba=_?QZ~OiZ96H`*o)A3`S!u=PZf^55<8_O|>EA@4bzqw2l1YQV zuW0ry7GJdaR5@DF6%E?M*wd;zHHid*szSUu37jT&GvPuR%Hv@h~ogJ&uG`2}^ha zfkC*74v+$Z=^6^P>n>VDft}0RFzHB)(&jK2kCKSSc>mhJCOhek82# z3(=2~GKLcMII$ zx>7%S_25BzPtAFS1ZYh>?HjDKJgw9E9y&Fh`>&V?rI#883U0rSnyU-iU8w;isz9k9 z^9{RUGc;<>UTw!@dLhQ@{ctYrBVZf0n&(cvP&*jh;*4y5j(BIpHcevC-J zI%49XE7-!1qsr1|9>!0F@E@-b%e*gj&wsNWPD0%ca|m-aBC~stffPah7^lWC%7I&i zd(+TTuKWa>CUW5|VOkrH8?en%owU(>U23NF(% z0`sEgxEZ-O=J$fleqJydMuiRs@p;0I*DHvFE04Z$MK}6=AT^YhID}Wlm2)Q-9!JeP z));RgE6utbmYVqhxh%2c#g`*3a-!o4Z%5$Gb`j`;_ipr+cB(KmXo{jky2?vmj->xE z)7Bj#2kH8c!ng5nIT$`9ruSQ)YfDmEl`Il?aiTEJNIE8eW#@O}r9G?8XnLoYTLU zxCBM-iB$ADLUwa4{SWsJo#iv9*y}j1+>>Uy-?4Oprrm))i9}BuVd5?w#UxhViyu#Q zfJY}=_TKs74$rc8vSmMq#wsmP7xT0NsB!4?-0P`L`!C>bANi|*%xf9F$SqmuM|SjL zG#a$>cJDfDiiHdmdG{AJ0&>#K9ed%+Rbxy$X= zT$?&l4|Z~$!_RZn`F2!xA;zCn^%2xLwb4Ti-$$!_d3}#Rqli|Vwe0-%`%LBQa_{kI z0NxDd@8=h{<0pcd$xPvj50_tGKC7m@_S#wQqsa4%E)VJ&uAN1ng5D6FwV}MSuC9Jo zRpTuBx^u9&y!MP(DE&Rn%oDQ~E%t|2s|aS}Ce zD#4>_j8??)q3#=t7gbb58|ic6_*!&LcvD5Rp1*m`*>dEg?~NDNH`K-ID(Y&A-yg4w zl~bl@WwbJrnUpN9t}CW5R$o>ZTUr;dHK~isYidHVx=;|R!Uw*imG7Dq6v7H3r)LT1 zUNsBSRaIXbZzxyMj>Im+sW^+%q#(R{^{Uk+p=%rJ;`NQ8jq%1^tMWwq6t=)3Nh zl8p4Z?}m+OGX&0nWXy~}&J99wakN66&K38><@|@g??IOB+km zHgP)Qche45%B@!;*DKq%5R#k~BG*ZWhN7T0v*UrQDQEb+{G6YZl#Iw{ zfWxtKHhF&kQ27mgPCY7U!-aqOz*XX^F+itC6;yl;Z`4rSNV`E$ zkJ{iizTp0i?ysS}B3yEgzGPX^GAeReZA|KAq<@n%*}WY%8p@J`;qs#8F1TTr20HTa z%LJg3k{OMKGa8FB<$L`XYSY!cF>tmm^_U~>j;NoK5|^oPId$Q*%gdqjXW57I6gsCXt!vETZ&Yt1 z?Fy$S@+#g&24#-))7<%7hDO?RhL2J5SyPpiaF#;p6RYcH*;MX?uV|nsS?MC%X!6*c z+!V8OrseY55xKyuB}>;VuBpcUIMZ=u8DZeeks!woI(hWu@q3z|$ByO7!3)EA*@?%A zIhvkX^_7qM*`kVl=8b0`|Q_Y(_ZY5!A-(u>Vox zDD|fH7sAoxSOY7g47-6;et%HDUWQBRFoDO0Q<%b)*H+35uEKR%n4HqIl*V`!yeZWo z?2==$mTM6L60D*E*HBzGLHTz+ok8`)cA5uxYHv7$nQ zShw}sDCKAI;ke|oAn%Bs+Fpkqx$=(G!}aIb->sqgns_W)$tHJ;6)1~RFU8bCHiiq` zfPjfLzFJJw>M+F%VUNC%88JqM=&Q?b&}@h24>p68OB>L=&N6w=y2J86l;?<{=SaKA zXkZO{I4bvpL3Jaxz&oVJ%VN;mNlw*+J4<$bt(B#4`dkOEC=j6>XE~b7t zlRuLIxmVUzR##ysLuw<`@DEpyTv>s+McNFb!_mj&0f!Ylhd-QrM2QU4`R1-y*DWgv zYnGF**L@>Hx`WXr;GFp~QHUESXUZ%UgAFf*a=EXE59iA8;r#i=HTilcHjL?;UQhX% z{wX5_#x@(U`hs-2JwK>He32%Yzg{yTtE9wa zz&x3!qISo~P_&_;u3=Je0&@$3F&|wWjM+40%(!DFOx%h`OVfbUe>zU2KZ}l?a@*L& zlku`JACz{XTx09joat9d`j_*hZO3OFt(_jZW{y={;J)FsZp`?E!4}bbB3ms*2&XIq!q|+CJbEV%c>3_B9 zM!ubrUS#cO()UXG_icV7{{cy-udL=Oe;V<{QEJl-{|hC3yiGU!>4j7}@}+N<^iSo< zf48I;9i|XT7Dzn-b17xkn}V1@Snz>5(L|9y5WDJq<`0@Tm2<{Y@YPZl73>I z^t&bf)I8}sCH+C0Zsgl5>9hxC+CE0U1Cma!CZy9%{%LrBoQ{j}I$N&v3nl$NB2xdR zYaG+?|0+r6H`X-Wl)qWhkF({Q^t%s{zEjeFmZ$ygmGqe9Z}dAL>E(I+XBz2^gMXN$ z@;iw2yHL_ULRgOUt0euKwtSO+v!s7CPyV|l{ro)XJ0<<M!Y2bLbyj zx_EI(sPNLu%8Ekg7SAa@J9PH!*>mU2Is4pD;cARUBjquN&ph{RtTh)m-ncPVehtpC z26x_|R}Imc^5R%@Q!FU1#Pf!r_?pJXptzyV-EeAj!_`#{wD?h6QMYj;R?~uFECNT1 zQTj}5l3kHr#T4Tm!6}41WCrcOl*K~1C|#$BJBw?i!BcSMW0W}MHX*shosY(e%9$we zR5_n6xHLIIgJ7#yt|s_p(FZ1o=aJ^8KAPl`kcTUGX!*k`sNgEU(k;M6uKaE6{6R3) zGpwKg0_68~BHmMa>2DA;OCs5iSZRJ0D*}wl_m7Sn98&Bh__Xgwx{pzoA4z^M`v}^% zfxx%D&0p>@@XMlm-}aQH{$nm9?U_Rdy$4!=|Gxf50n1hX`dg5IV~3aMm+z=X{w2Ec z!7KEp+K6M^2rD>}aXBR+`9{_-=`q&u;jttzs#n7Gp8Qjfae)LT1#qVNCHkd;eOb=L+-z$HB|B|a_hb2#@6?h^8QPMpTc`Xi5V=~~L9Q7q?IX)_RC z-I)i!D-ZsqJotlo@ZEWE`tD~gd7jUM{|xwXuuq@(6OX-t{ioo?QpG%`2KHyc4|??U zzA5oJpBgj{9%JK7-~7%c&)hsXeMmMJ{Z)DJiahvrz)7CJQbWeUV|tuFD)|267!G(W zkIRn>e$UAc=P^7kX)H*5{zM%D4j#MX+#&d}r!pMym>rk*34ZiD9nNEQTz*UNbKm7~ z9;4%Om*5BA?QkBO<8rs)$4_@SkI8ZQjNlbB9L{5HT)rUqwWnv}zZU$uGaSxiZ(Pz` zfb=?0=x`o$<8l&$7U4&q>2MxvXH%UO|u{F}q5d2YiNgOT!QXM7!+9)?%TEb@J`b*ffXC3l z?-KlX^iy*jJa)$U%Yt7{9T^TDGvoYq!GBP~up2AmlIAA4XL~u`w=b?@Pklgr3K|z@vhftakJiAL+P3@COB#Y6xg{O?-YQIFD^Xe}~}9 z)^hfM$F#Wog5dKmcQ}t_>5?C(R`6+c6wd}O;rBZ5R1f3Qx*Pcwq}9=?C?2C#MDYkw zUK+eCdc!Jy6K_=&FSB8lq_%D;UKPo_LWyNAT4$jb{wRSKWoYq9v7EdX(;J~dW2_SV zf~xukyZ}%YTzxfjzZx%;xc3J6z0u1s4qjE|gb1qK)BNi-OI1TO$|^0ctF4Vz;7vg+ zOUw15Ml9ksSOm%}gI?vN;Rr&T2=SL+wk%wGeU|Wx=*7u3Sk=a>CwA#5*oZZ;x{9Dt z^dQz){)P?HBCW!UNn{XLEWI(6z6-lILMYQ}dUJy=O8Kn}y1Ihik6>&Ky}Urs+GtG; zUo7Q!3kX_6uM5z{rO{Z9hvJm7DE(lZ?iSPIU%FU{N4tEn#68_5Y_)svN|)ewh5sZJ z{!&{Z+1KPqXZ}zi7R9PK>ZGp06$M(maLr`=RM1`F^)D6#PuVf?k;2{M4wP6KuV-v2 zD%^oWG2&T<;OQp}*VfcsQ(nUkm+@~Q;#GdSUV+ubjIr)!xmuEx_`^;@oW1#WW9H=yTsG@aL@MJ^OSN$x)_XM{l> zFCU=y@^Rd2aEfb+e@$@ZGh6N-G4wPvPu-4`~vA$XjnsOv;{34Uy(Cm$K${9 z`Hi6`y*A-L9gBz%2k{|%2i@TKf`|X7gM+sq&e1{1R<9ETSH0dM<-NnupMX3n&s7Gu zeCiBNV{)ax-r$zLRd6l$SSj}thTfL@w85zkmH(d&PICgq3uT;1`qB_g@y{EahG2?6 zU~pSsKNMW$)bZML9v%|;e`Wa4JWToj!Qj>oV`ZwM{3F6=g5V^-wZkz6w{k8wxaI!= zk3X%=)A3J+-tw<7xaI%2!D($y^?JtOmR{G6e197fdeyf}Md68-GqPxRrCY z!7cxCkH5zMDnoDiUuSU3|51b6`2RhFTl$|G+{*Kk;Ht02{~<$fI1f|K2>{pT9o%Cpqtqw)WJhTh6kYH+LXdXJCB&!~s1oVz_-b|cM9&eqe6pQ`R93jRQ`)RT;u<51Xp=9{ttP0xwOYuJU-gq-%6E) z! z^NGGT{#OdFdTIQxHS{+AcM7ihYW&~f;Tr!R_V{c3|GwdG#|e83ZuNT6C^x~z+; zJR1Mc@^H2D*@COSYUfJ@_w8J2a4XMxk55YM{53;wjhVR zwO`oe;YBjeyGd|g-))A!wg2Z0PM6y6Kj871CHxC%(ujlXX7!q9aLZ?-!Kq4>|7O8e z&WP~eV(4u=yv^X2|KkR?{D0)}-z5BhZ0IfjzX+~&en99a5D^Z=)z9DU;SUM@S`Y6Q zyxha}xx{7K@aZ`{1p%XuHdf;uJ%+vKS~~4`F?(!!L2=K2(Enkg@3z;kCE}~ zS3LYg!FxP>hTs!qVMOhDuHXe8{yxD^6kO%e@y~3*eZ4L)xYcWg$4AHaPYUkK^F0q& zdHOs)D$kJMzC5oQ+{!cI?ar@MFO}y5f~!2OVux}MzgzHYJw7VWErR>l{i;xKpZ|FVxBS<5e3buV9L>kDUl9)vWIk5y z;o6UU#KW~8`J~{!eg4JZR-Ug4u6|M|<$l-DQ&Z6RKamFaID9)yHMr&9EV$}BPx!Za z_!WYGQgD?=>-|2%-}==z4Q|VQ(&MvU_@7CcbI4y}aLea2dGPNT-1cAJ6I|tQlX9Oi z^c3>do_h>#%Y6?yB@VUQgTm*%9{#A{a|G9N)vs0v?)yWT!EL!wk59Mof6CBXeZTL~ z?-%-?7(Rz8w&|7`~%cIwN88-B`UdF#G z;Dh9`<*qb1g#@j~4;kFje=`sMOdkAIgIoE>z1zv7c3UrYo+voUY3pmU!L2->5M0~Q zCxy?aJ^W6=w+pW2Y8?Ki;ZIFg?eHCg+j76}@#zr$C%(t&H5Mmp&lv`{d_I;3?-X41 zdPex~@bEo?KP@o=q| zeI7oUGU9mE!`~+OaWh;x+287Wiova%?-g9_HdXk<4ZW?$R)bqUKg)w36kPSy{(Zvf zPEN%ym2xKwuJWiIrVH-tHPhf$o{Ky_rNaM4LvQs;7~JytWgdLo8QJoyUPpWQO;YaL z1o!nS6x`Qqj=`-wi#$GU!oSJTTfN#1Zu$Hs4}NT6wqCj}e3FMhBjrvL+}EpEa9^*v z2DkED>hT#6{>_5Z?F9UPzpRtCd3c%NpY-^I1n&^sm*+l%TY0|i@u?R+#}+yJ5Fcyj z(+qC;#PZ-@65QA8s~-M{l=}_A)y^6>o-+Jxd;fvKtvq`@KE1-f_{`jTEiky{^P=D? zkNW?hhwHp=*zlpHLG52Ay*E>??H}JExboNM0;d^z+dsa?;8xB~!H-5UI94D7==jjHw{O=H4?YvO<|BHuTE%@gI zSH0BFA2Ix`ox2Tg%kA^{Y!dz_ots2-#6!e_mQ-z9jJ;99Qw zXS3jbyGt0{miwSSG~UP@!uf) z`wYGHlM70moN9-yLchqvzajXgf~)*$hjPJvf2c9ImGfgBpGSp%v!SpY{ z{k(^l3x2QQs+T^8e%SCo)yVUGgIjri;qj>#{xu6+`MzHF8T=%}|FJyyQwF!?UUzYZ zKe?pZZTZ3sZuz(6!5_$jKWXq&f(-xX3~uE)X%UEUsQq=l>ogDloY-f&;A($u@8=8d z`_;t;x8<(#_;d;XPaAqG{}&Bz`Rwud>=!;S7<#G-weu?mxA7;qgw2qC?<+*X2FF_j zC%tU^nJu_)w|O4EPWUVkThV`OuQB|s-L5mZ)%Rl_ zpMK##Ah^m?c%suY<>6-u{$C!SiGl}9T)we5S$U2!xRvJ}f~(!;37?M&uJT+h_$?k@ zA$Xg|N9Fmf;J!RxGPsrJTOOZH!v9Z#t2`PX{^sGGLO(w2?4Ww7JSPe6%kxfyTX|-A ze7b~xgW$eAANBB;gudD1qw;)Oa9^Ig3~uH5s>dgwAdTbXrS9y@^KK76Pv{E;_w7?6 zxG&FQgIjslczhzlzfW*qo_-I%LFj+s@lpHi7u=WU&jz>h9CfL)gX+~R{8tFB^5{Cl zS`Y6O`VV@1RGu2aeR*y&xRvLV9-ke;|9gU~Ji5;CoQLZ=L%+vI*aW{5+?VGM2DkEz zi)7oUUl@HzaN=Xf1CI*s%lV{-9}qsh9)Gple;EEYF8$WvHlDxg^Le|I`_G2n#`C`! z-0B-%=JNS^Z4=zr>vJBySjzp9;ObYZ*F%QC)vMd!R=| zz3K&5y|xPfO&Hro$Gp#-CyD$zzdl}YU;fh!Zsnip@tG?8&o=Z{{<#LX^8dTwM~ghVPW3|% z*Y|>cEV#J8nGF;Fiz5f~)-NrM~X>@D9PhDY)8M$8SG0{H^@Y z8{C%rlE-I<@LzmcMt)2GPX?zUzmETF4Q}aQGWaEizIs)LzqQYe2Dg0v-Qc#|{RX%E zH+92k zYq?rqUo-q|eLZY&TkdX;kJi_Kp||?J?9sm{e)X!MxB8yAF1OuI698 z;8y;SW6$eFR5yAolndfd_=$4AUe1ckG%9HJ@FBVH@bJEI4(Rr9{f^auhwFE*f-x>Z z%hm5&>H7vyIWdi1*9yWYd~J5$XbuHTvJ@NoUkRJVugccunBT)#8LMuZ9~r+#ND zBt}pfh*Gu7$(K_4`oW9h4zx5K2x4dA@0i$h-39ezkb1kI%0}oz6L3K#VP9ppQS=({ znV4kVu~%^I<5WtD{-m+e$En|A3>0op?W8+J5>Gi=Yz2D}Wr`$Tc#O_n^oNsr!mrZx z&?!A!29=#D1Bp(US0gP?PrNcCQ3M++_hMt^ZftK^4=_R&ZFB8gooDkkyJ0H3z?c($Nu{SyU6 z>%LJ?RC-TAQRGtvMGLnicTriXEB?y$Q_$186Ss5>u7?jqk`J+X$z3FaXxlmw$&r>_ zbPD2ntfY!DtvHaC-2Nw&+u zJ|_^QJ2?0Zf))%o6?O>W9#(DP3v5)*=YofW3yb2v8;Q2H&Z5%VTlYh-o?tfdmz#y- z-Q6}yinNDAk;EkvB{9;zt(5X5N~bjMA`UBJlikIY1+DiGlb+VufN>SHuo;8+yev(& zqnn7dw=?hLHdZ^YgSkU@eSZa6cMPxIT zXl@q69fven8)qL{p!Pa1EC1|63gq&Oxe6X;Zf#=+|AvfC%rynAUx1iZJ*^A4ZdK6i znjJo*pj8QGkgsMp9a2(ZprSs)?5a?+tJAVBPy69Q$Vx|UY3XxYqgOQCh*;#4w)2qE z3WJOv5Jxf#l)`D6m?&?I+_)F8>>)^(-m zh}Nfj(nMKTV&#MAjQWWTQ4c$+ z@RsntF(`I^_`!nP>!{L_;QAJDxE@Fxf_uEJdco3u|%yT8s>99-qC8^rpIt6)pf{?k<&XbcO z?PXKYRZ{0kl&14cOGkQ;AP7ZFlHA7PBp)J$(NaV?7m92yZjb5I2vN7}qEir?iz})` z*B-e#ldVi4?UAx&jVlO%36mV~5VqU^hfnE-xDX97g&{56r>sA*a?gqcHNZIjH>Jds zIT~Ne8*ieebVJ=>a5pI}qxxjip48wUc(j#nWwaFIHv!H238b|&?Fr(i!;K*T8L^Fe zpX5peg0caKwO^VA+SU+e7kkFUF}qOOiR>S_Xk9y;kq#a-7+;wpo^Tys6&lVa!rkE8 zowgEWhSugCn+!_I(LZ=UDNiji9l%gV(|`?Nr=`=y&pBCw-%xEPwUc?V z!j6FilhGN>%rKCEK^7whE_9?)b5J25Vu5k)Ppeo86)``-+y;s}5;)P@*G3CVaT ztCk4g-_mq{5IaX&RI==TuN9f>J*^a@iRT~=J;w~129d|@}otk8Oeorgs!wA{Z=}<$&!vF%jp!-b@0i0Orq7MB*I%E zJO0P}onk%V0TM6L{!l&HG1(+I)(n`?;cA3|p?9=ENukbCR9Y=^eVKGhhWD@wkIyZw z>pxvjn=P$N*~O11PowEyp*>+0FV$;x^WcXF|O50(J#FLRHemx$#pvxIQDVGUTEm)#LhWX(=n|okK z(yynr6n|+?5aa}t)^)graTBd;ak0B~J^rFT;>~ExL>X!4fdvbu4Z7s@HRw!p8^Ro+ zsh4>k%53-Sb_mG1>o&g5Pn!Y+Ben-93CWk@rC#t2|Euu@d-sX<%H`PN{qmI%0z1T` zTDWws2^hR`Qm}Z%qBUzu7p;xJF16QJH`LWq{e`YCZ>Xla38mB7p4dPEo7v7>LIM}F zAjS3gEc`{mjK&#_!HQ*Y#@U8T3;B1ig8Nw1^^pCf&*q|d@f-S+ga3t+eovnK zS4sM1)VF&1|6ki1?#t?EWJ??vjb$D5K7x-CLG!yE**q8C_%|VB)`<@0J+SEVbo?hA zBd@F@<&~?6epzQBfq44)^yU@-aW6sTSK0-*$d!N7+Z_e1 zWZJ{ePd}aWbt3vMQVa*pWayxGDyTVW{;34wh~U3h1mpXKpqTG6e|%bcCCbN$`ZV*r zE|YsCExq3`!uBSAxy_7gzbv}M_d&bQJ9gXYskXOlr_X)1|+y?;r zfZ#7kh9~7*EBIZ)ietW{{XO=KYXL8+SO1`eom}{*^5FO8!SBz5@5+PIVpOhjf0PIR zO&*-yy`jCFjuYqRz7W{3;6EmZ#zFb%m<;<8{nx0WDMc_UUm3?ft-*({-{>xApuUy{gUc}XjaCLc^u<0$ zD|v*!76besiv*)%>_8O@*Tx!dTvi*az6Be&XpubmS&k81nKcHoc8jcIJ4k%kE0e-nS~})<|x&zO0gDbA#~otN>eN**&l< zuA78O4%*%;YvV2%nB!d%?6OYVgUS9{Ifv+!9>wNcy02EPY^(()wDA;9vAXL5e>5HN zW?4L6u;fzmY&~sKpM$o?eICFS388{{@>(G z^e5rJ@_*RiB>#H3=Xo-&iI26z(~?enXvm{{bnio-{w%ahI()k=FgOieln<>8)8X@3 zVd!nS>kMw~qkHL5xz?W7=Ao}QxV7hI!PTA-mM#cxGxXM;yA8dyo9=n(+u`SikF~=~ zdH4(&dTWO&VvK{j6T1Ao+<#xrbfUJSb}kr)8ytk&cZ@HV#zTDQ@_PKIqe^h1xAv(M zTL9q$&|CX-8Qk*e@%VHKpC1@{+kQ`xe$bcaOuc{DrW#}!R=LJ{09ZMN; zOqa87w=IGz{}&y3aGRmG_Iy_Qcc0I13~uePPjKa*5dMEM^wyrI$+*I|!%V@I&yesr z+t6D(e97Qe9{r}B^3fsY6NcW(bMmpSAf+!6f35KF<$`bUaBa7fWE?_z(HL3%_I$yW zzqaT3hCXEIe`x5bJubw5I-WE1HlF|1;O_>VmitG8Tl@Ug@F!Po6TK!75e|~y+UFR- zNq#HO+YG&J*9#0i@qbA8FE;e=0j_eE8JsQ^w|jOFpKjq(ZRo8(=w2SaoQ;Ow%DGK& za#P|zDEzt8#XZ?!I{vFX)2J{UT8{^W{&c~Keg^(4z1@R@_>Un19CHo5wZkO_C;B>} zf5G6i7NPuKG58qg;h4z2!gC;FeEC9{gd07lN0{x!d5je(hcr zr0*2bx8KlPKm57ie*OO1(A)YQOPwnYVq1j&D$g2&TY0V&T^`ELflRPNV%xXzQCJzVF>9UiXpz-|xMdEJ1A>u+#0 bzN(x$KMYCxP+aGe3q4%tf$Ke7=XL)dL#AO} diff --git a/hdfs/setup.sh b/hdfs/setup.sh new file mode 100644 index 000000000..ac69b525d --- /dev/null +++ b/hdfs/setup.sh @@ -0,0 +1,7 @@ +export USE_HDFS=1 +export LD_LIBRARY_PATH=$JAVA_HOME/jre/lib/amd64/server:$JAVA_HOME/jre/lib/amd64:/usr/lib/hadoop/lib/native + +export CLASSPATH= +for f in `find /usr/lib/hadoop-hdfs | grep jar`; do export CLASSPATH=$CLASSPATH:$f; done +for f in `find /usr/lib/hadoop | grep jar`; do export CLASSPATH=$CLASSPATH:$f; done +for f in `find /usr/lib/hadoop/client | grep jar`; do export CLASSPATH=$CLASSPATH:$f; done diff --git a/util/env_hdfs.cc b/util/env_hdfs.cc index eb2b12cba..09ee03438 100644 --- a/util/env_hdfs.cc +++ b/util/env_hdfs.cc @@ -15,11 +15,11 @@ #include #include "rocksdb/env.h" #include "rocksdb/status.h" -#include "hdfs/hdfs.h" #include "hdfs/env_hdfs.h" #define HDFS_EXISTS 0 -#define HDFS_DOESNT_EXIST 1 +#define HDFS_DOESNT_EXIST -1 +#define HDFS_SUCCESS 0 // // This file defines an HDFS environment for rocksdb. It uses the libhdfs @@ -223,7 +223,7 @@ class HdfsWritableFile: public WritableFile { if (hdfsFlush(fileSys_, hfile_) == -1) { return IOError(filename_, errno); } - if (hdfsSync(fileSys_, hfile_) == -1) { + if (hdfsHSync(fileSys_, hfile_) == -1) { return IOError(filename_, errno); } Log(mylog, "[hdfs] HdfsWritableFile Synced %s\n", filename_.c_str()); @@ -398,12 +398,40 @@ Status HdfsEnv::NewRandomRWFile(const std::string& fname, return Status::NotSupported("NewRandomRWFile not supported on HdfsEnv"); } +class HdfsDirectory : public Directory { + public: + explicit HdfsDirectory(int fd) : fd_(fd) {} + ~HdfsDirectory() { + //close(fd_); + } + + virtual Status Fsync() { + //if (fsync(fd_) == -1) { + // return IOError("directory", errno); + // } + return Status::OK(); + } + + private: + int fd_; +}; + Status HdfsEnv::NewDirectory(const std::string& name, unique_ptr* result) { - return Status::NotSupported("NewDirectory not supported on HdfsEnv"); + + int value = hdfsCreateDirectory(fileSys_, name.c_str()); + result->reset(new HdfsDirectory(0)); + switch (value) { + case HDFS_SUCCESS: // directory created + return Status::OK(); + default: + Log(mylog, "directory already exists "); + return Status::OK(); + } } bool HdfsEnv::FileExists(const std::string& fname) { + int value = hdfsExists(fileSys_, fname.c_str()); switch (value) { case HDFS_EXISTS: @@ -412,8 +440,8 @@ bool HdfsEnv::FileExists(const std::string& fname) { return false; default: // anything else should be an error Log(mylog, "FileExists hdfsExists call failed"); - throw HdfsFatalException("hdfsExists call failed with error " + - std::to_string(value) + ".\n"); + throw HdfsFatalException("1. hdfsExists call failed with error " + + std::to_string(value) + " on path " + fname + ".\n"); } } @@ -449,13 +477,13 @@ Status HdfsEnv::GetChildren(const std::string& path, default: // anything else should be an error Log(mylog, "GetChildren hdfsExists call failed"); throw HdfsFatalException("hdfsExists call failed with error " + - std::to_string(value) + ".\n"); + std::to_string(value) + " on path " + path.c_str() + ".\n"); } return Status::OK(); } Status HdfsEnv::DeleteFile(const std::string& fname) { - if (hdfsDelete(fileSys_, fname.c_str()) == 0) { + if (hdfsDelete(fileSys_, fname.c_str(),1) == 0) { return Status::OK(); } return IOError(fname, errno); @@ -478,7 +506,7 @@ Status HdfsEnv::CreateDirIfMissing(const std::string& name) { return CreateDir(name); default: // anything else should be an error Log(mylog, "CreateDirIfMissing hdfsExists call failed"); - throw HdfsFatalException("hdfsExists call failed with error " + + throw HdfsFatalException("3. hdfsExists call failed with error " + std::to_string(value) + ".\n"); } }; @@ -514,7 +542,7 @@ Status HdfsEnv::GetFileModificationTime(const std::string& fname, // target already exists. So, we delete the target before attemting the // rename. Status HdfsEnv::RenameFile(const std::string& src, const std::string& target) { - hdfsDelete(fileSys_, target.c_str()); + hdfsDelete(fileSys_, target.c_str(), 1); if (hdfsRename(fileSys_, src.c_str(), target.c_str()) == 0) { return Status::OK(); } From d788bb8f719c16cfcb789d8656555315783b9f56 Mon Sep 17 00:00:00 2001 From: Mike Orr Date: Wed, 21 May 2014 07:50:37 -0400 Subject: [PATCH 2/8] - hdfs cleanup; fix to NewDirectory to comply with definition in env.h - fix compile error with env_test; static casts added --- util/env_hdfs.cc | 37 ++++++++++++++++--------------------- util/env_test.cc | 8 ++++---- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/util/env_hdfs.cc b/util/env_hdfs.cc index 09ee03438..6b6d56c00 100644 --- a/util/env_hdfs.cc +++ b/util/env_hdfs.cc @@ -401,16 +401,9 @@ Status HdfsEnv::NewRandomRWFile(const std::string& fname, class HdfsDirectory : public Directory { public: explicit HdfsDirectory(int fd) : fd_(fd) {} - ~HdfsDirectory() { - //close(fd_); - } + ~HdfsDirectory() {} - virtual Status Fsync() { - //if (fsync(fd_) == -1) { - // return IOError("directory", errno); - // } - return Status::OK(); - } + virtual Status Fsync() { return Status::OK(); } private: int fd_; @@ -418,20 +411,21 @@ class HdfsDirectory : public Directory { Status HdfsEnv::NewDirectory(const std::string& name, unique_ptr* result) { - - int value = hdfsCreateDirectory(fileSys_, name.c_str()); - result->reset(new HdfsDirectory(0)); + int value = hdfsExists(fileSys_, name.c_str()); switch (value) { - case HDFS_SUCCESS: // directory created - return Status::OK(); - default: - Log(mylog, "directory already exists "); + case HDFS_EXISTS: + result->reset(new HdfsDirectory(0)); return Status::OK(); + default: // fail if the directory doesn't exist + Log(mylog, "NewDirectory hdfsExists call failed"); + throw HdfsFatalException("hdfsExists call failed with error " + + std::to_string(value) + " on path " + name + + ".\n"); } } bool HdfsEnv::FileExists(const std::string& fname) { - + int value = hdfsExists(fileSys_, fname.c_str()); switch (value) { case HDFS_EXISTS: @@ -440,8 +434,9 @@ bool HdfsEnv::FileExists(const std::string& fname) { return false; default: // anything else should be an error Log(mylog, "FileExists hdfsExists call failed"); - throw HdfsFatalException("1. hdfsExists call failed with error " + - std::to_string(value) + " on path " + fname + ".\n"); + throw HdfsFatalException("hdfsExists call failed with error " + + std::to_string(value) + " on path " + fname + + ".\n"); } } @@ -477,13 +472,13 @@ Status HdfsEnv::GetChildren(const std::string& path, default: // anything else should be an error Log(mylog, "GetChildren hdfsExists call failed"); throw HdfsFatalException("hdfsExists call failed with error " + - std::to_string(value) + " on path " + path.c_str() + ".\n"); + std::to_string(value) + ".\n"); } return Status::OK(); } Status HdfsEnv::DeleteFile(const std::string& fname) { - if (hdfsDelete(fileSys_, fname.c_str(),1) == 0) { + if (hdfsDelete(fileSys_, fname.c_str(), 1) == 0) { return Status::OK(); } return IOError(fname, errno); diff --git a/util/env_test.cc b/util/env_test.cc index 2abce6f3a..c0d00ce94 100644 --- a/util/env_test.cc +++ b/util/env_test.cc @@ -285,7 +285,7 @@ TEST(EnvPosixTest, DecreaseNumBgThreads) { // Increase to 5 threads. Task 0 and 2 running. env_->SetBackgroundThreads(5, Env::Priority::HIGH); Env::Default()->SleepForMicroseconds(kDelayMicros); - ASSERT_EQ(0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_EQ((unsigned int)0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); ASSERT_TRUE(tasks[0].IsSleeping()); ASSERT_TRUE(tasks[2].IsSleeping()); @@ -330,7 +330,7 @@ TEST(EnvPosixTest, DecreaseNumBgThreads) { tasks[4].WakeUp(); Env::Default()->SleepForMicroseconds(kDelayMicros); - ASSERT_EQ(0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_EQ((unsigned int)0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); for (size_t i = 5; i < 8; i++) { ASSERT_TRUE(tasks[i].IsSleeping()); } @@ -360,13 +360,13 @@ TEST(EnvPosixTest, DecreaseNumBgThreads) { env_->Schedule(&SleepingBackgroundTask::DoSleepTask, &tasks[9], Env::Priority::HIGH); Env::Default()->SleepForMicroseconds(kDelayMicros); - ASSERT_GT(env_->GetThreadPoolQueueLen(Env::Priority::HIGH), 0); + ASSERT_GT(env_->GetThreadPoolQueueLen(Env::Priority::HIGH), (unsigned int)0); ASSERT_TRUE(!tasks[8].IsSleeping() || !tasks[9].IsSleeping()); // Increase to 4 threads. Task 5, 8, 9 running. env_->SetBackgroundThreads(4, Env::Priority::HIGH); Env::Default()->SleepForMicroseconds(kDelayMicros); - ASSERT_EQ(0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); + ASSERT_EQ((unsigned int)0, env_->GetThreadPoolQueueLen(Env::Priority::HIGH)); ASSERT_TRUE(tasks[8].IsSleeping()); ASSERT_TRUE(tasks[9].IsSleeping()); From 591f71285c1349fc6b6d2a113b360d0d2da46cf6 Mon Sep 17 00:00:00 2001 From: Mike Orr Date: Wed, 21 May 2014 07:54:22 -0400 Subject: [PATCH 3/8] cleanup exception text --- util/env_hdfs.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/env_hdfs.cc b/util/env_hdfs.cc index 6b6d56c00..1618e5468 100644 --- a/util/env_hdfs.cc +++ b/util/env_hdfs.cc @@ -501,7 +501,7 @@ Status HdfsEnv::CreateDirIfMissing(const std::string& name) { return CreateDir(name); default: // anything else should be an error Log(mylog, "CreateDirIfMissing hdfsExists call failed"); - throw HdfsFatalException("3. hdfsExists call failed with error " + + throw HdfsFatalException("hdfsExists call failed with error " + std::to_string(value) + ".\n"); } }; From 81b498bc159d675014aaf89b56a341c1b2dffd46 Mon Sep 17 00:00:00 2001 From: Chilledheart Date: Thu, 22 May 2014 01:24:42 +0800 Subject: [PATCH 4/8] Print pthread_t in a more safe way --- util/env_posix.cc | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/util/env_posix.cc b/util/env_posix.cc index 7ffba6f53..1f8c3bcf2 100644 --- a/util/env_posix.cc +++ b/util/env_posix.cc @@ -94,6 +94,17 @@ static Status IOError(const std::string& context, int err_number) { return Status::IOError(context, strerror(err_number)); } +// TODO(sdong): temp logging. Need to help debugging. Remove it when +// the feature is proved to be stable. +inline void PrintThreadInfo(size_t thread_id, pthread_t id) { + unsigned char* ptc = (unsigned char*)(void*)(&id); + fprintf(stdout, "Bg thread %zu terminates 0x", thread_id); + for (size_t i = 0; i < sizeof(id); i++) { + fprintf(stdout, "%02x", (unsigned)(ptc[i])); + } + fprintf(stdout, "\n"); +} + #ifdef NDEBUG // empty in release build #define TEST_KILL_RANDOM(rocksdb_kill_odds) @@ -1514,8 +1525,7 @@ class PosixEnv : public Env { PthreadCall("unlock", pthread_mutex_unlock(&mu_)); // TODO(sdong): temp logging. Need to help debugging. Remove it when // the feature is proved to be stable. - fprintf(stdout, "Bg thread %zu terminates %llx\n", thread_id, - static_cast(terminating_thread)); + PrintThreadInfo(thread_id, terminating_thread); break; } void (*function)(void*) = queue_.front().function; From 6de6a066313876c0142db643a75272c3578b39f6 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 21 May 2014 11:43:35 -0700 Subject: [PATCH 5/8] FIFO compaction style Summary: Introducing new compaction style -- FIFO. FIFO compaction style has write amplification of 1 (+1 for WAL) and it deletes the oldest files when the total DB size exceeds pre-configured values. FIFO compaction style is suited for storing high-frequency event logs. Test Plan: Added a unit test Reviewers: dhruba, haobo, sdong Reviewed By: dhruba Subscribers: alberts, leveldb Differential Revision: https://reviews.facebook.net/D18765 --- HISTORY.md | 1 + db/column_family.cc | 22 +++++++++--- db/compaction.cc | 8 ++++- db/compaction.h | 9 ++++- db/compaction_picker.cc | 71 +++++++++++++++++++++++++++++++++++++++ db/compaction_picker.h | 22 ++++++++++-- db/db_impl.cc | 38 +++++++++++++++++---- db/db_impl_debug.cc | 3 +- db/db_test.cc | 64 ++++++++++++++++++++++++++++++----- db/version_set.cc | 22 ++++++++---- db/version_set.h | 1 + include/rocksdb/options.h | 17 ++++++++-- util/options.cc | 3 ++ 13 files changed, 248 insertions(+), 33 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index f9bdd7c98..43025b722 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -7,6 +7,7 @@ ### New Features * Hash index for block-based table will be materialized and reconstructed more efficiently. Previously hash index is constructed by scanning the whole table during every table open. +* FIFO compaction style ## 3.0.0 (05/05/2014) diff --git a/db/column_family.cc b/db/column_family.cc index 39c37b9e8..9cf0c0d49 100644 --- a/db/column_family.cc +++ b/db/column_family.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include "db/db_impl.h" #include "db/version_set.h" @@ -116,6 +117,15 @@ ColumnFamilyOptions SanitizeOptions(const InternalKeyComparator* icmp, collector_factories.push_back( std::make_shared()); + if (result.compaction_style == kCompactionStyleFIFO) { + result.num_levels = 1; + // since we delete level0 files in FIFO compaction when there are too many + // of them, these options don't really mean anything + result.level0_file_num_compaction_trigger = std::numeric_limits::max(); + result.level0_slowdown_writes_trigger = std::numeric_limits::max(); + result.level0_stop_writes_trigger = std::numeric_limits::max(); + } + return result; } @@ -196,7 +206,7 @@ ColumnFamilyData::ColumnFamilyData(const std::string& dbname, uint32_t id, options_(*db_options, SanitizeOptions(&internal_comparator_, &internal_filter_policy_, options)), mem_(nullptr), - imm_(options.min_write_buffer_number_to_merge), + imm_(options_.min_write_buffer_number_to_merge), super_version_(nullptr), super_version_number_(0), local_sv_(new ThreadLocalPtr(&SuperVersionUnrefHandle)), @@ -209,16 +219,20 @@ ColumnFamilyData::ColumnFamilyData(const std::string& dbname, uint32_t id, // if dummy_versions is nullptr, then this is a dummy column family. if (dummy_versions != nullptr) { - internal_stats_.reset(new InternalStats(options.num_levels, db_options->env, - db_options->statistics.get())); + internal_stats_.reset(new InternalStats( + options_.num_levels, db_options->env, db_options->statistics.get())); table_cache_.reset( new TableCache(dbname, &options_, storage_options, table_cache)); if (options_.compaction_style == kCompactionStyleUniversal) { compaction_picker_.reset( new UniversalCompactionPicker(&options_, &internal_comparator_)); - } else { + } else if (options_.compaction_style == kCompactionStyleLevel) { compaction_picker_.reset( new LevelCompactionPicker(&options_, &internal_comparator_)); + } else { + assert(options_.compaction_style == kCompactionStyleFIFO); + compaction_picker_.reset( + new FIFOCompactionPicker(&options_, &internal_comparator_)); } Log(options_.info_log, "Options for column family \"%s\":\n", diff --git a/db/compaction.cc b/db/compaction.cc index 962ce1232..a8caa59ef 100644 --- a/db/compaction.cc +++ b/db/compaction.cc @@ -29,7 +29,8 @@ static uint64_t TotalFileSize(const std::vector& files) { Compaction::Compaction(Version* input_version, int level, int out_level, uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes, - bool seek_compaction, bool enable_compression) + bool seek_compaction, bool enable_compression, + bool deletion_compaction) : level_(level), out_level_(out_level), max_output_file_size_(target_file_size), @@ -39,6 +40,7 @@ Compaction::Compaction(Version* input_version, int level, int out_level, cfd_(input_version_->cfd_), seek_compaction_(seek_compaction), enable_compression_(enable_compression), + deletion_compaction_(deletion_compaction), grandparent_index_(0), seen_key_(false), overlapped_bytes_(0), @@ -83,6 +85,8 @@ bool Compaction::IsTrivialMove() const { TotalFileSize(grandparents_) <= max_grandparent_overlap_bytes_); } +bool Compaction::IsDeletionCompaction() const { return deletion_compaction_; } + void Compaction::AddInputDeletions(VersionEdit* edit) { for (int which = 0; which < 2; which++) { for (size_t i = 0; i < inputs_[which].size(); i++) { @@ -92,6 +96,7 @@ void Compaction::AddInputDeletions(VersionEdit* edit) { } bool Compaction::IsBaseLevelForKey(const Slice& user_key) { + assert(cfd_->options()->compaction_style != kCompactionStyleFIFO); if (cfd_->options()->compaction_style == kCompactionStyleUniversal) { return bottommost_level_; } @@ -155,6 +160,7 @@ void Compaction::MarkFilesBeingCompacted(bool value) { // Is this compaction producing files at the bottommost level? void Compaction::SetupBottomMostLevel(bool isManual) { + assert(cfd_->options()->compaction_style != kCompactionStyleFIFO); if (cfd_->options()->compaction_style == kCompactionStyleUniversal) { // If universal compaction style is used and manual // compaction is occuring, then we are guaranteed that diff --git a/db/compaction.h b/db/compaction.h index 8fd95f909..aaa402303 100644 --- a/db/compaction.h +++ b/db/compaction.h @@ -54,6 +54,9 @@ class Compaction { // moving a single input file to the next level (no merging or splitting) bool IsTrivialMove() const; + // If true, just delete all files in inputs_[0] + bool IsDeletionCompaction() const; + // Add all inputs to this compaction as delete operations to *edit. void AddInputDeletions(VersionEdit* edit); @@ -91,11 +94,13 @@ class Compaction { private: friend class CompactionPicker; friend class UniversalCompactionPicker; + friend class FIFOCompactionPicker; friend class LevelCompactionPicker; Compaction(Version* input_version, int level, int out_level, uint64_t target_file_size, uint64_t max_grandparent_overlap_bytes, - bool seek_compaction = false, bool enable_compression = true); + bool seek_compaction = false, bool enable_compression = true, + bool deletion_compaction = false); int level_; int out_level_; // levels to which output files are stored @@ -108,6 +113,8 @@ class Compaction { bool seek_compaction_; bool enable_compression_; + // if true, just delete files in inputs_[0] + bool deletion_compaction_; // Each compaction reads inputs from "level_" and "level_+1" std::vector inputs_[2]; // The two sets of inputs diff --git a/db/compaction_picker.cc b/db/compaction_picker.cc index a8700bbbc..3416a0bac 100644 --- a/db/compaction_picker.cc +++ b/db/compaction_picker.cc @@ -9,6 +9,8 @@ #include "db/compaction_picker.h" +#define __STDC_FORMAT_MACROS +#include #include #include "util/log_buffer.h" #include "util/statistics.h" @@ -307,6 +309,9 @@ Compaction* CompactionPicker::CompactRange(Version* version, int input_level, const InternalKey* begin, const InternalKey* end, InternalKey** compaction_end) { + // CompactionPickerFIFO has its own implementation of compact range + assert(options_->compaction_style != kCompactionStyleFIFO); + std::vector inputs; bool covering_the_whole_range = true; @@ -886,4 +891,70 @@ Compaction* UniversalCompactionPicker::PickCompactionUniversalSizeAmp( return c; } +Compaction* FIFOCompactionPicker::PickCompaction(Version* version, + LogBuffer* log_buffer) { + assert(version->NumberLevels() == 1); + uint64_t total_size = 0; + for (const auto& file : version->files_[0]) { + total_size += file->file_size; + } + + if (total_size <= options_->compaction_options_fifo.max_table_files_size || + version->files_[0].size() == 0) { + // total size not exceeded + LogToBuffer(log_buffer, + "[%s] FIFO compaction: nothing to do. Total size %" PRIu64 + ", max size %" PRIu64 "\n", + version->cfd_->GetName().c_str(), total_size, + options_->compaction_options_fifo.max_table_files_size); + return nullptr; + } + + if (compactions_in_progress_[0].size() > 0) { + LogToBuffer(log_buffer, + "[%s] FIFO compaction: Already executing compaction. No need " + "to run parallel compactions since compactions are very fast", + version->cfd_->GetName().c_str()); + return nullptr; + } + + Compaction* c = new Compaction(version, 0, 0, 0, 0, false, false, + true /* is deletion compaction */); + // delete old files (FIFO) + for (auto ritr = version->files_[0].rbegin(); + ritr != version->files_[0].rend(); ++ritr) { + auto f = *ritr; + total_size -= f->file_size; + c->inputs_[0].push_back(f); + char tmp_fsize[16]; + AppendHumanBytes(f->file_size, tmp_fsize, sizeof(tmp_fsize)); + LogToBuffer(log_buffer, "[%s] FIFO compaction: picking file %" PRIu64 + " with size %s for deletion", + version->cfd_->GetName().c_str(), f->number, tmp_fsize); + if (total_size <= options_->compaction_options_fifo.max_table_files_size) { + break; + } + } + + c->MarkFilesBeingCompacted(true); + compactions_in_progress_[0].insert(c); + + return c; +} + +Compaction* FIFOCompactionPicker::CompactRange(Version* version, + int input_level, + int output_level, + const InternalKey* begin, + const InternalKey* end, + InternalKey** compaction_end) { + assert(input_level == 0); + assert(output_level == 0); + *compaction_end = nullptr; + LogBuffer log_buffer(InfoLogLevel::INFO_LEVEL, options_->info_log.get()); + auto c = PickCompaction(version, &log_buffer); + log_buffer.FlushBufferToLog(); + return c; +} + } // namespace rocksdb diff --git a/db/compaction_picker.h b/db/compaction_picker.h index 6527ef967..65b1bc37a 100644 --- a/db/compaction_picker.h +++ b/db/compaction_picker.h @@ -47,9 +47,10 @@ class CompactionPicker { // compaction_end will be set to nullptr. // Client is responsible for compaction_end storage -- when called, // *compaction_end should point to valid InternalKey! - Compaction* CompactRange(Version* version, int input_level, int output_level, - const InternalKey* begin, const InternalKey* end, - InternalKey** compaction_end); + virtual Compaction* CompactRange(Version* version, int input_level, + int output_level, const InternalKey* begin, + const InternalKey* end, + InternalKey** compaction_end); // Free up the files that participated in a compaction void ReleaseCompactionFiles(Compaction* c, Status status); @@ -162,4 +163,19 @@ class LevelCompactionPicker : public CompactionPicker { Compaction* PickCompactionBySize(Version* version, int level, double score); }; +class FIFOCompactionPicker : public CompactionPicker { + public: + FIFOCompactionPicker(const Options* options, + const InternalKeyComparator* icmp) + : CompactionPicker(options, icmp) {} + + virtual Compaction* PickCompaction(Version* version, + LogBuffer* log_buffer) override; + + virtual Compaction* CompactRange(Version* version, int input_level, + int output_level, const InternalKey* begin, + const InternalKey* end, + InternalKey** compaction_end) override; +}; + } // namespace rocksdb diff --git a/db/db_impl.cc b/db/db_impl.cc index bdc1832dc..f87442767 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -1590,7 +1590,7 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family, return s; } - int max_level_with_files = 1; + int max_level_with_files = 0; { MutexLock l(&mutex_); Version* base = cfd->current(); @@ -1604,6 +1604,7 @@ Status DBImpl::CompactRange(ColumnFamilyHandle* column_family, // in case the compaction is unversal or if we're compacting the // bottom-most level, the output level will be the same as input one if (cfd->options()->compaction_style == kCompactionStyleUniversal || + cfd->options()->compaction_style == kCompactionStyleFIFO || level == max_level_with_files) { s = RunManualCompaction(cfd, level, level, begin, end); } else { @@ -1754,14 +1755,16 @@ Status DBImpl::RunManualCompaction(ColumnFamilyData* cfd, int input_level, // For universal compaction, we enforce every manual compaction to compact // all files. if (begin == nullptr || - cfd->options()->compaction_style == kCompactionStyleUniversal) { + cfd->options()->compaction_style == kCompactionStyleUniversal || + cfd->options()->compaction_style == kCompactionStyleFIFO) { manual.begin = nullptr; } else { begin_storage = InternalKey(*begin, kMaxSequenceNumber, kValueTypeForSeek); manual.begin = &begin_storage; } if (end == nullptr || - cfd->options()->compaction_style == kCompactionStyleUniversal) { + cfd->options()->compaction_style == kCompactionStyleUniversal || + cfd->options()->compaction_style == kCompactionStyleFIFO) { manual.end = nullptr; } else { end_storage = InternalKey(*end, 0, static_cast(0)); @@ -2150,6 +2153,24 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, if (!c) { // Nothing to do LogToBuffer(log_buffer, "Compaction nothing to do"); + } else if (c->IsDeletionCompaction()) { + // TODO(icanadi) Do we want to honor snapshots here? i.e. not delete old + // file if there is alive snapshot pointing to it + assert(c->num_input_files(1) == 0); + assert(c->level() == 0); + assert(c->column_family_data()->options()->compaction_style == + kCompactionStyleFIFO); + for (const auto& f : *c->inputs(0)) { + c->edit()->DeleteFile(c->level(), f->number); + } + status = versions_->LogAndApply(c->column_family_data(), c->edit(), &mutex_, + db_directory_.get()); + InstallSuperVersion(c->column_family_data(), deletion_state); + LogToBuffer(log_buffer, "[%s] Deleted %d files\n", + c->column_family_data()->GetName().c_str(), + c->num_input_files(0)); + c->ReleaseCompactionFiles(status); + *madeProgress = true; } else if (!is_manual && c->IsTrivialMove()) { // Move file to next level assert(c->num_input_files(0) == 1); @@ -2219,8 +2240,9 @@ Status DBImpl::BackgroundCompaction(bool* madeProgress, if (!m->done) { // We only compacted part of the requested range. Update *m // to the range that is left to be compacted. - // Universal compaction should always compact the whole range + // Universal and FIFO compactions should always compact the whole range assert(m->cfd->options()->compaction_style != kCompactionStyleUniversal); + assert(m->cfd->options()->compaction_style != kCompactionStyleFIFO); m->tmp_storage = *manual_end; m->begin = &m->tmp_storage; } @@ -4468,13 +4490,15 @@ Status DB::Open(const DBOptions& db_options, const std::string& dbname, if (s.ok()) { for (auto cfd : *impl->versions_->GetColumnFamilySet()) { - if (cfd->options()->compaction_style == kCompactionStyleUniversal) { + if (cfd->options()->compaction_style == kCompactionStyleUniversal || + cfd->options()->compaction_style == kCompactionStyleFIFO) { Version* current = cfd->current(); for (int i = 1; i < current->NumberLevels(); ++i) { int num_files = current->NumLevelFiles(i); if (num_files > 0) { - s = Status::InvalidArgument("Not all files are at level 0. Cannot " - "open with universal compaction style."); + s = Status::InvalidArgument( + "Not all files are at level 0. Cannot " + "open with universal or FIFO compaction style."); break; } } diff --git a/db/db_impl_debug.cc b/db/db_impl_debug.cc index d6551b45a..927a01a04 100644 --- a/db/db_impl_debug.cc +++ b/db/db_impl_debug.cc @@ -81,7 +81,8 @@ Status DBImpl::TEST_CompactRange(int level, const Slice* begin, cfd = cfh->cfd(); } int output_level = - (cfd->options()->compaction_style == kCompactionStyleUniversal) + (cfd->options()->compaction_style == kCompactionStyleUniversal || + cfd->options()->compaction_style == kCompactionStyleFIFO) ? level : level + 1; return RunManualCompaction(cfd, level, output_level, begin, end); diff --git a/db/db_test.cc b/db/db_test.cc index 05403fc07..5e30b33f7 100644 --- a/db/db_test.cc +++ b/db/db_test.cc @@ -317,6 +317,7 @@ class DBTest { kCompressedBlockCache, kInfiniteMaxOpenFiles, kxxHashChecksum, + kFIFOCompaction, kEnd }; int option_config_; @@ -339,7 +340,8 @@ class DBTest { kSkipPlainTable = 8, kSkipHashIndex = 16, kSkipNoSeekToLast = 32, - kSkipHashCuckoo = 64 + kSkipHashCuckoo = 64, + kSkipFIFOCompaction = 128, }; DBTest() : option_config_(kDefault), @@ -391,6 +393,10 @@ class DBTest { if ((skip_mask & kSkipHashCuckoo) && (option_config_ == kHashCuckoo)) { continue; } + if ((skip_mask & kSkipFIFOCompaction) && + option_config_ == kFIFOCompaction) { + continue; + } break; } @@ -503,6 +509,10 @@ class DBTest { options.table_factory.reset(NewBlockBasedTableFactory(table_options)); break; } + case kFIFOCompaction: { + options.compaction_style = kCompactionStyleFIFO; + break; + } case kBlockBasedTableWithPrefixHashIndex: { BlockBasedTableOptions table_options; table_options.index_type = BlockBasedTableOptions::kHashSearch; @@ -1394,7 +1404,7 @@ TEST(DBTest, GetEncountersEmptyLevel) { env_->SleepForMicroseconds(1000000); ASSERT_EQ(NumTableFilesAtLevel(0, 1), 1); // XXX - } while (ChangeOptions(kSkipUniversalCompaction)); + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction)); } // KeyMayExist can lead to a few false positives, but not false negatives. @@ -1460,7 +1470,8 @@ TEST(DBTest, KeyMayExist) { // KeyMayExist function only checks data in block caches, which is not used // by plain table format. - } while (ChangeOptions(kSkipPlainTable | kSkipHashIndex)); + } while ( + ChangeOptions(kSkipPlainTable | kSkipHashIndex | kSkipFIFOCompaction)); } TEST(DBTest, NonBlockingIteration) { @@ -4387,7 +4398,8 @@ TEST(DBTest, ApproximateSizes) { ASSERT_GT(NumTableFilesAtLevel(1, 1), 0); } // ApproximateOffsetOf() is not yet implemented in plain table format. - } while (ChangeOptions(kSkipUniversalCompaction | kSkipPlainTable)); + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction | + kSkipPlainTable)); } TEST(DBTest, ApproximateSizes_MixOfSmallAndLarge) { @@ -4531,8 +4543,8 @@ TEST(DBTest, HiddenValuesAreRemoved) { // ApproximateOffsetOf() is not yet implemented in plain table format, // which is used by Size(). // skip HashCuckooRep as it does not support snapshot - } while (ChangeOptions(kSkipUniversalCompaction | kSkipPlainTable | - kSkipHashCuckoo)); + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction | + kSkipPlainTable | kSkipHashCuckoo)); } TEST(DBTest, CompactBetweenSnapshots) { @@ -4588,7 +4600,7 @@ TEST(DBTest, CompactBetweenSnapshots) { ASSERT_EQ("sixth", Get(1, "foo")); ASSERT_EQ(AllEntriesFor("foo", 1), "[ sixth ]"); // skip HashCuckooRep as it does not support snapshot - } while (ChangeOptions(kSkipHashCuckoo)); + } while (ChangeOptions(kSkipHashCuckoo | kSkipFIFOCompaction)); } TEST(DBTest, DeletionMarkers1) { @@ -4694,7 +4706,7 @@ TEST(DBTest, OverlapInLevel0) { Flush(1); ASSERT_EQ("3", FilesPerLevel(1)); ASSERT_EQ("NOT_FOUND", Get(1, "600")); - } while (ChangeOptions(kSkipUniversalCompaction)); + } while (ChangeOptions(kSkipUniversalCompaction | kSkipFIFOCompaction)); } TEST(DBTest, L0_CompactionBug_Issue44_a) { @@ -6797,6 +6809,42 @@ TEST(DBTest, ChecksumTest) { ASSERT_EQ("f", Get("e")); ASSERT_EQ("h", Get("g")); } + +TEST(DBTest, FIFOCompactionTest) { + for (int iter = 0; iter < 2; ++iter) { + // first iteration -- auto compaction + // second iteration -- manual compaction + Options options; + options.compaction_style = kCompactionStyleFIFO; + options.write_buffer_size = 100 << 10; // 100KB + options.compaction_options_fifo.max_table_files_size = 500 << 10; // 500KB + options.compression = kNoCompression; + options.create_if_missing = true; + if (iter == 1) { + options.disable_auto_compactions = true; + } + DestroyAndReopen(&options); + + Random rnd(301); + for (int i = 0; i < 6; ++i) { + for (int j = 0; j < 100; ++j) { + ASSERT_OK(Put(std::to_string(i * 100 + j), RandomString(&rnd, 1024))); + } + // flush should happen here + } + if (iter == 0) { + ASSERT_OK(dbfull()->TEST_WaitForCompact()); + } else { + ASSERT_OK(db_->CompactRange(nullptr, nullptr)); + } + // only 5 files should survive + ASSERT_EQ(NumTableFilesAtLevel(0), 5); + for (int i = 0; i < 50; ++i) { + // these keys should be deleted in previous compaction + ASSERT_EQ("NOT_FOUND", Get(std::to_string(i))); + } + } +} } // namespace rocksdb int main(int argc, char** argv) { diff --git a/db/version_set.cc b/db/version_set.cc index 02e9aa152..5327cf55f 100644 --- a/db/version_set.cc +++ b/db/version_set.cc @@ -711,7 +711,8 @@ void Version::ComputeCompactionScore( int max_score_level = 0; int num_levels_to_check = - (cfd_->options()->compaction_style != kCompactionStyleUniversal) + (cfd_->options()->compaction_style != kCompactionStyleUniversal && + cfd_->options()->compaction_style != kCompactionStyleFIFO) ? NumberLevels() - 1 : 1; @@ -730,14 +731,18 @@ void Version::ComputeCompactionScore( // setting, or very high compression ratios, or lots of // overwrites/deletions). int numfiles = 0; + uint64_t total_size = 0; for (unsigned int i = 0; i < files_[level].size(); i++) { if (!files_[level][i]->being_compacted) { + total_size += files_[level][i]->file_size; numfiles++; } } - - // If we are slowing down writes, then we better compact that first - if (numfiles >= cfd_->options()->level0_stop_writes_trigger) { + if (cfd_->options()->compaction_style == kCompactionStyleFIFO) { + score = static_cast(total_size) / + cfd_->options()->compaction_options_fifo.max_table_files_size; + } else if (numfiles >= cfd_->options()->level0_stop_writes_trigger) { + // If we are slowing down writes, then we better compact that first score = 1000000; } else if (numfiles >= cfd_->options()->level0_slowdown_writes_trigger) { score = 10000; @@ -803,6 +808,10 @@ bool CompareSeqnoDescending(const Version::Fsize& first, } // anonymous namespace void Version::UpdateFilesBySize() { + if (cfd_->options()->compaction_style == kCompactionStyleFIFO) { + // don't need this + return; + } // No need to sort the highest level because it is never compacted. int max_level = (cfd_->options()->compaction_style == kCompactionStyleUniversal) @@ -871,7 +880,8 @@ bool Version::NeedsCompaction() const { // TODO(sdong): improve this function to be accurate for universal // compactions. int num_levels_to_check = - (cfd_->options()->compaction_style != kCompactionStyleUniversal) + (cfd_->options()->compaction_style != kCompactionStyleUniversal && + cfd_->options()->compaction_style != kCompactionStyleFIFO) ? NumberLevels() - 1 : 1; for (int i = 0; i < num_levels_to_check; i++) { @@ -1253,7 +1263,7 @@ struct VersionSet::ManifestWriter { class VersionSet::Builder { private: // Helper to sort v->files_ - // kLevel0LevelCompaction -- NewestFirst + // kLevel0LevelCompaction -- NewestFirst (also used for FIFO compaction) // kLevel0UniversalCompaction -- NewestFirstBySeqNo // kLevelNon0 -- BySmallestKey struct FileComparator { diff --git a/db/version_set.h b/db/version_set.h index 13a138341..ffadb5813 100644 --- a/db/version_set.h +++ b/db/version_set.h @@ -217,6 +217,7 @@ class Version { friend class CompactionPicker; friend class LevelCompactionPicker; friend class UniversalCompactionPicker; + friend class FIFOCompactionPicker; class LevelFileNumIterator; class LevelFileIteratorState; diff --git a/include/rocksdb/options.h b/include/rocksdb/options.h index e26ecde51..9ba6a522c 100644 --- a/include/rocksdb/options.h +++ b/include/rocksdb/options.h @@ -53,8 +53,18 @@ enum CompressionType : char { }; enum CompactionStyle : char { - kCompactionStyleLevel = 0x0, // level based compaction style - kCompactionStyleUniversal = 0x1 // Universal compaction style + kCompactionStyleLevel = 0x0, // level based compaction style + kCompactionStyleUniversal = 0x1, // Universal compaction style + kCompactionStyleFIFO = 0x2, // FIFO compaction style +}; + +struct CompactionOptionsFIFO { + // once the total sum of table files reaches this, we will delete the oldest + // table file + // Default: 1GB + uint64_t max_table_files_size; + + CompactionOptionsFIFO() : max_table_files_size(1 * 1024 * 1024 * 1024) {} }; // Compression options for different compression algorithms like Zlib @@ -429,6 +439,9 @@ struct ColumnFamilyOptions { // The options needed to support Universal Style compactions CompactionOptionsUniversal compaction_options_universal; + // The options for FIFO compaction style + CompactionOptionsFIFO compaction_options_fifo; + // Use KeyMayExist API to filter deletes when this is true. // If KeyMayExist returns false, i.e. the key definitely does not exist, then // the delete is a noop. KeyMayExist only incurs in-memory look up. diff --git a/util/options.cc b/util/options.cc index 22952f587..4fe8b219e 100644 --- a/util/options.cc +++ b/util/options.cc @@ -135,6 +135,7 @@ ColumnFamilyOptions::ColumnFamilyOptions(const Options& options) compaction_style(options.compaction_style), verify_checksums_in_compaction(options.verify_checksums_in_compaction), compaction_options_universal(options.compaction_options_universal), + compaction_options_fifo(options.compaction_options_fifo), filter_deletes(options.filter_deletes), max_sequential_skip_in_iterations( options.max_sequential_skip_in_iterations), @@ -413,6 +414,8 @@ void ColumnFamilyOptions::Dump(Logger* log) const { Log(log, "Options.compaction_options_universal.compression_size_percent: %u", compaction_options_universal.compression_size_percent); + Log(log, "Options.compaction_options_fifo.max_table_files_size: %" PRIu64, + compaction_options_fifo.max_table_files_size); std::string collector_names; for (const auto& collector_factory : table_properties_collector_factories) { collector_names.append(collector_factory->Name()); From 1fd4654de5346f646079215ae4788f5c4e5b801f Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Wed, 21 May 2014 15:25:05 -0700 Subject: [PATCH 6/8] Update HISTORY.md --- HISTORY.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/HISTORY.md b/HISTORY.md index 43025b722..9b014c1f2 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -1,6 +1,6 @@ # Rocksdb Change Log -## Unreleased (3.1.0) +## 3.1.0 (05/21/2014) ### Public API changes * Replaced ColumnFamilyOptions::table_properties_collectors with ColumnFamilyOptions::table_properties_collector_factories From 3a1cf1281b94dc672e58b40c9c838e0ec5cb5a5c Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Thu, 22 May 2014 10:24:24 -0700 Subject: [PATCH 7/8] Run FIFO compaction as part of db_crashtest2.py Summary: As title Test Plan: ran it Reviewers: dhruba Reviewed By: dhruba Differential Revision: https://reviews.facebook.net/D18783 --- tools/db_crashtest2.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/tools/db_crashtest2.py b/tools/db_crashtest2.py index 0a12b5a60..3ef383afc 100644 --- a/tools/db_crashtest2.py +++ b/tools/db_crashtest2.py @@ -60,7 +60,7 @@ def main(argv): + str(ops_per_thread) + "\nwrite_buffer_size=" \ + str(write_buf_size) + "\n" - total_check_mode = 3 + total_check_mode = 4 check_mode = 0 while time.time() < exit_time: @@ -75,8 +75,14 @@ def main(argv): # normal run with universal compaction mode additional_opts = "--ops_per_thread=" + str(ops_per_thread) + \ " --compaction_style=1" + elif check_mode == 2: + # normal run with FIFO compaction mode + # ops_per_thread is divided by 5 because FIFO compaction + # style is quite a bit slower on reads with lot of files + additional_opts = "--ops_per_thread=" + str(ops_per_thread / 5) + \ + " --compaction_style=2" else: - # nomral run + # normal run additional_opts = "--ops_per_thread=" + str(ops_per_thread) dbname = tempfile.mkdtemp(prefix='rocksdb_crashtest_') From f068d2a94dc93c43b917fc1a4ab3612e6d50f2e6 Mon Sep 17 00:00:00 2001 From: Igor Canadi Date: Fri, 23 May 2014 10:27:56 -0700 Subject: [PATCH 8/8] Move master version to 3.2 --- Makefile | 2 +- include/rocksdb/version.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 1dc741e5c..c148aee7e 100644 --- a/Makefile +++ b/Makefile @@ -146,7 +146,7 @@ SHARED = $(SHARED1) else # Update db.h if you change these. SHARED_MAJOR = 3 -SHARED_MINOR = 0 +SHARED_MINOR = 2 SHARED1 = ${LIBNAME}.$(PLATFORM_SHARED_EXT) SHARED2 = $(SHARED1).$(SHARED_MAJOR) SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR) diff --git a/include/rocksdb/version.h b/include/rocksdb/version.h index e440412eb..2bae1ed6e 100644 --- a/include/rocksdb/version.h +++ b/include/rocksdb/version.h @@ -6,7 +6,7 @@ // Also update Makefile if you change these #define ROCKSDB_MAJOR 3 -#define ROCKSDB_MINOR 1 +#define ROCKSDB_MINOR 2 #define ROCKSDB_PATCH 0 // Do not use these. We made the mistake of declaring macros starting with