NextGraph
/
lmdb
mirror of https://github.com/nextgraph-org/lmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change default mem init behavior

Browse Source 
Init malloc'd memory by default, use #MDB_NOMEMINIT to disable.
Bump version to 0.9.10
vmware
Howard Chu 11 years ago
parent 92f71f50f9
commit 509a197728
2 changed files with 35 additions and 28 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 51
      libraries/liblmdb/lmdb.h
  2. 12
      libraries/liblmdb/mdb.c

51
libraries/liblmdb/lmdb.h
Unescape View File

@ -70,11 +70,15 @@
* access to locks and lock file. Exceptions: On read-only filesystems * access to locks and lock file. Exceptions: On read-only filesystems
* or with the #MDB_NOLOCK flag described under #mdb_env_open(). * or with the #MDB_NOLOCK flag described under #mdb_env_open().
* *
* - By default, unused portions of the datafile may receive garbage data * - By default, in versions before 0.9.10, unused portions of the data
* from memory freed by other code. (This does not happen when using * file might receive garbage data from memory freed by other code.
* the #MDB_WRITEMAP flag.) Applications handling sensitive data * (This did not happen when using the #MDB_WRITEMAP flag.) As of
* 0.9.10 the default behavior is to initialize such memory before
* writing to the data file. Since there may be a slight performance
* cost due to this initialization, applications may disable it using
* the #MDB_NOMEMINIT flag. Applications handling sensitive data
* which must not be written, and which don't use #MDB_WRITEMAP, * which must not be written, and which don't use #MDB_WRITEMAP,
* need to prevent this with the #MDB_CLEANMEM flag. * should not use this flag.
* *
* - A thread can only use one transaction at a time, plus any child * - A thread can only use one transaction at a time, plus any child
* transactions. Each transaction belongs to one thread. See below. * transactions. Each transaction belongs to one thread. See below.
@ -180,7 +184,7 @@ typedef int mdb_filehandle_t;
/** Library minor version */ /** Library minor version */
#define MDB_VERSION_MINOR 9 #define MDB_VERSION_MINOR 9
/** Library patch version */ /** Library patch version */
#define MDB_VERSION_PATCH 9 #define MDB_VERSION_PATCH 10
/** Combine args a,b,c into a single integer for easy version comparisons */ /** Combine args a,b,c into a single integer for easy version comparisons */
#define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c))
@ -190,7 +194,7 @@ typedef int mdb_filehandle_t;
MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH)
/** The release date of this library version */ /** The release date of this library version */
#define MDB_VERSION_DATE "October 24, 2013" #define MDB_VERSION_DATE "November 11, 2013"
/** A stringifier for the version info */ /** A stringifier for the version info */
#define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")" #define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")"
@ -283,8 +287,8 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
#define MDB_NOLOCK 0x400000 #define MDB_NOLOCK 0x400000
/** don't do readahead (no effect on Windows) */ /** don't do readahead (no effect on Windows) */
#define MDB_NORDAHEAD 0x800000 #define MDB_NORDAHEAD 0x800000
/** don't write uninitialized malloc'd memory to datafile */ /** don't initialize malloc'd memory before writing to datafile */
#define MDB_CLEANMEM 0x1000000 #define MDB_NOMEMINIT 0x1000000
/** @} */ /** @} */
/** @defgroup mdb_dbi_open Database Flags /** @defgroup mdb_dbi_open Database Flags
@ -554,22 +558,25 @@ int mdb_env_create(MDB_env **env);
* supports it. Turning it off may help random read performance * supports it. Turning it off may help random read performance
* when the DB is larger than RAM and system RAM is full. * when the DB is larger than RAM and system RAM is full.
* The option is not implemented on Windows. * The option is not implemented on Windows.
* <li>#MDB_CLEANMEM * <li>#MDB_NOMEMINIT
* Don't write uninitialized memory to unused spaces in the datafile. * Don't initialize malloc'd memory before writing to unused spaces
* By default, memory for pages written to the datafile is obtained * in the data file. By default, memory for pages written to the data
* using malloc, and only the portions that LMDB uses are modified. * file is obtained using malloc. While these pages may be reused in
* Unused portions of a page may contain leftover data from other * subsequent transactions, freshly malloc'd pages will be initialized
* code that used the heap and subsequently freed that memory. * to zeroes before use. This avoids persisting leftover data from other
* That can be a problem for applications which handle sensitive data * code (that used the heap and subsequently freed the memory) into the
* like passwords, and it makes memory checkers like Valgrind noisy. * data file. Note that many other system libraries may allocate
* With this flag, unused portions of pages will be initialized to * and free memory from the heap for arbitrary uses. E.g., stdio may
* zero. This flag is not needed with #MDB_WRITEMAP, which writes * use the heap for file I/O buffers. This initialization step comes
* directly to the mmap instead of using malloc for pages. The * at some performance cost so some applications may want to disable
* it using this flag. This option can be a problem for applications
* which handle sensitive data like passwords, and it makes memory
* checkers like Valgrind noisy. This flag is not needed with #MDB_WRITEMAP,
* which writes directly to the mmap instead of using malloc for pages. The
* initialization is also skipped if #MDB_RESERVE is used; the * initialization is also skipped if #MDB_RESERVE is used; the
* caller is expected to overwrite all of the memory that was * caller is expected to overwrite all of the memory that was
* reserved in that case. * reserved in that case.
* This flag may be changed at any time using #mdb_env_set_flags(). * This flag may be changed at any time using #mdb_env_set_flags().
* It comes at some performance cost.
* </ul> * </ul>
* @param[in] mode The UNIX permissions to set on created files. This parameter * @param[in] mode The UNIX permissions to set on created files. This parameter
* is ignored on Windows. * is ignored on Windows.
@ -1155,8 +1162,8 @@ int mdb_get(MDB_txn *txn, MDB_dbi dbi, MDB_val *key, MDB_val *data);
* reserved space, which the caller can fill in later - before * reserved space, which the caller can fill in later - before
* the next update operation or the transaction ends. This saves * the next update operation or the transaction ends. This saves
* an extra memcpy if the data is being generated later. * an extra memcpy if the data is being generated later.
* MDB does nothing else with this memory, even if #MDB_CLEANMEM is * MDB does nothing else with this memory, the caller is expected
* set - the caller is expected to modify all of the space requested. * to modify all of the space requested.
* <li>#MDB_APPEND - append the given key/data pair to the end of the * <li>#MDB_APPEND - append the given key/data pair to the end of the
* database. No key comparisons are performed. This option allows * database. No key comparisons are performed. This option allows
* fast bulk loading when keys are already known to be in the * fast bulk loading when keys are already known to be in the

12
libraries/liblmdb/mdb.c
Unescape View File

@ -1323,7 +1323,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num)
MDB_env *env = txn->mt_env; MDB_env *env = txn->mt_env;
MDB_page *ret = env->me_dpages; MDB_page *ret = env->me_dpages;
size_t psize = env->me_psize, sz = psize, off; size_t psize = env->me_psize, sz = psize, off;
/* For #MDB_CLEANMEM, psize counts how much to init. /* For ! #MDB_NOMEMINIT, psize counts how much to init.
* For a single page alloc, we init everything after the page header. * For a single page alloc, we init everything after the page header.
* For multi-page, we init the final page; if the caller needed that * For multi-page, we init the final page; if the caller needed that
* many pages they will be filling in at least up to the last page. * many pages they will be filling in at least up to the last page.
@ -1341,7 +1341,7 @@ mdb_page_malloc(MDB_txn *txn, unsigned num)
off = sz - psize; off = sz - psize;
} }
if ((ret = malloc(sz)) != NULL) { if ((ret = malloc(sz)) != NULL) {
if (env->me_flags & MDB_CLEANMEM) { if (!(env->me_flags & MDB_NOMEMINIT)) {
memset((char *)ret + off, 0, psize); memset((char *)ret + off, 0, psize);
ret->mp_pad = 0; ret->mp_pad = 0;
} }
@ -2508,9 +2508,9 @@ mdb_freelist_save(MDB_txn *txn)
return rc; return rc;
} }
/* MDB_RESERVE cancels CLEANMEM in ovpage malloc (when no WRITEMAP) */ /* MDB_RESERVE cancels meminit in ovpage malloc (when no WRITEMAP) */
clean_limit = (env->me_flags & (MDB_CLEANMEM|MDB_WRITEMAP)) == MDB_CLEANMEM clean_limit = (env->me_flags & (MDB_NOMEMINIT|MDB_WRITEMAP))
? maxfree_1pg : SSIZE_MAX; ? SSIZE_MAX : maxfree_1pg;
for (;;) { for (;;) {
/* Come back here after each Put() in case freelist changed */ /* Come back here after each Put() in case freelist changed */
@ -3965,7 +3965,7 @@ fail:
* at runtime. Changing other flags requires closing the * at runtime. Changing other flags requires closing the
* environment and re-opening it with the new flags. * environment and re-opening it with the new flags.
*/ */
#define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_CLEANMEM) #define CHANGEABLE (MDB_NOSYNC|MDB_NOMETASYNC|MDB_MAPASYNC|MDB_NOMEMINIT)
#define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \ #define CHANGELESS (MDB_FIXEDMAP|MDB_NOSUBDIR|MDB_RDONLY|MDB_WRITEMAP| \
MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD) MDB_NOTLS|MDB_NOLOCK|MDB_NORDAHEAD)

Write Preview
Loading…
Cancel
Save