From 509a1977284a8d8c3affea9fc1afb9fa70ddb533 Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Mon, 11 Nov 2013 15:09:32 -0800 Subject: [PATCH] Change default mem init behavior Init malloc'd memory by default, use #MDB_NOMEMINIT to disable. Bump version to 0.9.10 --- libraries/liblmdb/lmdb.h | 51 +++++++++++++++++++++++----------------- libraries/liblmdb/mdb.c | 12 +++++----- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 98c4d45..5877e95 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -70,11 +70,15 @@ * access to locks and lock file. Exceptions: On read-only filesystems * or with the #MDB_NOLOCK flag described under #mdb_env_open(). * - * - By default, unused portions of the datafile may receive garbage data - * from memory freed by other code. (This does not happen when using - * the #MDB_WRITEMAP flag.) Applications handling sensitive data + * - By default, in versions before 0.9.10, unused portions of the data + * file might receive garbage data from memory freed by other code. + * (This did not happen when using the #MDB_WRITEMAP flag.) As of + * 0.9.10 the default behavior is to initialize such memory before + * writing to the data file. Since there may be a slight performance + * cost due to this initialization, applications may disable it using + * the #MDB_NOMEMINIT flag. Applications handling sensitive data * which must not be written, and which don't use #MDB_WRITEMAP, - * need to prevent this with the #MDB_CLEANMEM flag. + * should not use this flag. * * - A thread can only use one transaction at a time, plus any child * transactions. Each transaction belongs to one thread. See below. @@ -180,7 +184,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 9 +#define MDB_VERSION_PATCH 10 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -190,7 +194,7 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "October 24, 2013" +#define MDB_VERSION_DATE "November 11, 2013" /** A stringifier for the version info */ #define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")" @@ -283,8 +287,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel #define MDB_NOLOCK 0x400000 /** don't do readahead (no effect on Windows) */ #define MDB_NORDAHEAD 0x800000 - /** don't write uninitialized malloc'd memory to datafile */ -#define MDB_CLEANMEM 0x1000000 + /** don't initialize malloc'd memory before writing to datafile */ +#define MDB_NOMEMINIT 0x1000000 /** @} */ /** @defgroup mdb_dbi_open Database Flags @@ -554,22 +558,25 @@ int mdb_env_create(MDB_env **env); * supports it. Turning it off may help random read performance * when the DB is larger than RAM and system RAM is full. * The option is not implemented on Windows. - *
  • #MDB_CLEANMEM - * Don't write uninitialized memory to unused spaces in the datafile. - * By default, memory for pages written to the datafile is obtained - * using malloc, and only the portions that LMDB uses are modified. - * Unused portions of a page may contain leftover data from other - * code that used the heap and subsequently freed that memory. - * That can be a problem for applications which handle sensitive data - * like passwords, and it makes memory checkers like Valgrind noisy. - * With this flag, unused portions of pages will be initialized to - * zero. This flag is not needed with #MDB_WRITEMAP, which writes - * directly to the mmap instead of using malloc for pages. The + *
  • #MDB_NOMEMINIT + * Don't initialize malloc'd memory before writing to unused spaces + * in the data file. By default, memory for pages written to the data + * file is obtained using malloc. While these pages may be reused in + * subsequent transactions, freshly malloc'd pages will be initialized + * to zeroes before use. This avoids persisting leftover data from other + * code (that used the heap and subsequently freed the memory) into the + * data file. Note that many other system libraries may allocate + * and free memory from the heap for arbitrary uses. E.g., stdio may + * use the heap for file I/O buffers. This initialization step comes + * at some performance cost so some applications may want to disable + * it using this flag. This option can be a problem for applications + * which handle sensitive data like passwords, and it makes memory + * checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP, + * which writes directly to the mmap instead of using malloc for pages. The * initialization is also skipped if #MDB_RESERVE is used; the * caller is expected to overwrite all of the memory that was * reserved in that case. * This flag may be changed at any time using #mdb_env_set_flags(). - * It comes at some performance cost. * * @param[in] mode The UNIX permissions to set on created files. This parameter * is ignored on Windows. @@ -1155,8 +1162,8 @@ int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data); * reserved space, which the caller can fill in later - before * the next update operation or the transaction ends. This saves * an extra memcpy if the data is being generated later. - * MDB does nothing else with this memory, even if #MDB_CLEANMEM is - * set - the caller is expected to modify all of the space requested. + * MDB does nothing else with this memory, the caller is expected + * to modify all of the space requested. *
  • #MDB_APPEND - append the given key/data pair to the end of the * database. No key comparisons are performed. This option allows * fast bulk loading when keys are already known to be in the diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 2ef3fe4..1935cec 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -1323,7 +1323,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) MDB_env *env = txn->mt_env; MDB_page *ret = env->me_dpages; size_t psize = env->me_psize, sz = psize, off; - /* For #MDB_CLEANMEM, psize counts how much to init. + /* For ! #MDB_NOMEMINIT, psize counts how much to init. * For a single page alloc, we init everything after the page header. * For multi-page, we init the final page; if the caller needed that * many pages they will be filling in at least up to the last page. @@ -1341,7 +1341,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num) off = sz - psize; } if ((ret = malloc(sz)) != NULL) { - if (env->me_flags & MDB_CLEANMEM) { + if (!(env->me_flags & MDB_NOMEMINIT)) { memset((char *)ret + off, 0, psize); ret->mp_pad = 0; } @@ -2508,9 +2508,9 @@ mdb_freelist_save(MDB_txn *txn) return rc; } - /* MDB_RESERVE cancels CLEANMEM in ovpage malloc (when no WRITEMAP) */ - clean_limit = (env->me_flags & (MDB_CLEANMEM|MDB_WRITEMAP)) == MDB_CLEANMEM - ? maxfree_1pg : SSIZE_MAX; + /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */ + clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP)) + ? SSIZE_MAX : maxfree_1pg; for (;;) { /* Come back here after each Put() in case freelist changed */ @@ -3965,7 +3965,7 @@ fail: * at runtime. Changing other flags requires closing the * environment and re-opening it with the new flags. */ -#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_CLEANMEM) +#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT) #define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)