NextGraph
/
lmdb
mirror of https://github.com/nextgraph-org/lmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Allow mdb_env_set_mapsize() on an open environment

Browse Source 
The caller is responsible for making sure no transactions are
active in this process before resizing. This is slightly lighter
weight than doing a full env_close/env_open cycle.
vmware
Howard Chu 11 years ago
parent 2a28686e4f
commit 2e7130cab0
2 changed files with 118 additions and 85 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 17
      libraries/liblmdb/lmdb.h
  2. 186
      libraries/liblmdb/mdb.c

17
libraries/liblmdb/lmdb.h
Unescape View File

@ -662,8 +662,15 @@ int mdb_env_get_path(MDB_env *env, const char **path);
* 10485760 bytes. The size of the memory map is also the maximum size * 10485760 bytes. The size of the memory map is also the maximum size
* of the database. The value should be chosen as large as possible, * of the database. The value should be chosen as large as possible,
* to accommodate future growth of the database. * to accommodate future growth of the database.
* This function may only be called after #mdb_env_create() and before #mdb_env_open(). * This function should be called after #mdb_env_create() and before #mdb_env_open().
* The size may be changed by closing and reopening the environment. * It may be called at later times if no transactions are active in
* this process. Note that the library does not check for this condition,
* the caller must ensure it explicitly.
*
* If the mapsize is changed by another process, #mdb_txn_begin() will
* return #MDB_MAP_RESIZED. This function may be called with a size
* of zero to adopt the new size.
*
* Any attempt to set a size smaller than the space already consumed * Any attempt to set a size smaller than the space already consumed
* by the environment will be silently changed to the current size of the used space. * by the environment will be silently changed to the current size of the used space.
* @param[in] env An environment handle returned by #mdb_env_create() * @param[in] env An environment handle returned by #mdb_env_create()
@ -671,7 +678,8 @@ int mdb_env_get_path(MDB_env *env, const char **path);
* @return A non-zero error value on failure and 0 on success. Some possible * @return A non-zero error value on failure and 0 on success. Some possible
* errors are: * errors are:
* <ul> * <ul>
* <li>EINVAL - an invalid parameter was specified, or the environment is already open. * <li>EINVAL - an invalid parameter was specified, or the environment has
* an active write transaction.
* </ul> * </ul>
*/ */
int mdb_env_set_mapsize(MDB_env *env, size_t size); int mdb_env_set_mapsize(MDB_env *env, size_t size);
@ -757,7 +765,8 @@ int mdb_env_get_maxkeysize(MDB_env *env);
* <li>#MDB_PANIC - a fatal error occurred earlier and the environment * <li>#MDB_PANIC - a fatal error occurred earlier and the environment
* must be shut down. * must be shut down.
* <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's * <li>#MDB_MAP_RESIZED - another process wrote data beyond this MDB_env's
* mapsize and the environment must be shut down. * mapsize and this environment's map must be resized as well.
* See #mdb_env_set_mapsize().
* <li>#MDB_READERS_FULL - a read-only transaction was requested and * <li>#MDB_READERS_FULL - a read-only transaction was requested and
* the reader lock table is full. See #mdb_env_set_maxreaders(). * the reader lock table is full. See #mdb_env_set_maxreaders().
* <li>ENOMEM - out of memory. * <li>ENOMEM - out of memory.

186
libraries/liblmdb/mdb.c
Unescape View File

@ -3053,7 +3053,7 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
DPUTS("writing new meta page"); DPUTS("writing new meta page");
GET_PAGESIZE(psize); psize = env->me_psize;
meta->mm_magic = MDB_MAGIC; meta->mm_magic = MDB_MAGIC;
meta->mm_version = MDB_DATA_VERSION; meta->mm_version = MDB_DATA_VERSION;
@ -3238,11 +3238,97 @@ mdb_env_create(MDB_env **env)
return MDB_SUCCESS; return MDB_SUCCESS;
} }
static int
mdb_env_map(MDB_env *env, void *addr, int newsize)
{
MDB_page *p;
unsigned int flags = env->me_flags;
#ifdef _WIN32
int rc;
HANDLE mh;
LONG sizelo, sizehi;
sizelo = env->me_mapsize & 0xffffffff;
sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */
/* Windows won't create mappings for zero length files.
* Just allocate the maxsize right now.
*/
if (newsize) {
if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
|| !SetEndOfFile(env->me_fd)
|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0)
return ErrCode();
}
mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
PAGE_READWRITE : PAGE_READONLY,
sizehi, sizelo, NULL);
if (!mh)
return ErrCode();
env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
FILE_MAP_WRITE : FILE_MAP_READ,
0, 0, env->me_mapsize, addr);
rc = env->me_map ? 0 : ErrCode();
CloseHandle(mh);
if (rc)
return rc;
#else
int prot = PROT_READ;
if (flags & MDB_WRITEMAP) {
prot |= PROT_WRITE;
if (newsize && ftruncate(env->me_fd, env->me_mapsize) < 0)
return ErrCode();
}
env->me_map = mmap(addr, env->me_mapsize, prot, MAP_SHARED,
env->me_fd, 0);
if (env->me_map == MAP_FAILED) {
env->me_map = NULL;
return ErrCode();
}
/* Turn off readahead. It's harmful when the DB is larger than RAM. */
#ifdef MADV_RANDOM
madvise(env->me_map, env->me_mapsize, MADV_RANDOM);
#else
#ifdef POSIX_MADV_RANDOM
posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM);
#endif /* POSIX_MADV_RANDOM */
#endif /* MADV_RANDOM */
#endif /* _WIN32 */
/* Can happen because the address argument to mmap() is just a
* hint. mmap() can pick another, e.g. if the range is in use.
* The MAP_FIXED flag would prevent that, but then mmap could
* instead unmap existing pages to make room for the new map.
*/
if (addr && env->me_map != addr)
return EBUSY; /* TODO: Make a new MDB_* error code? */
p = (MDB_page *)env->me_map;
env->me_metas[0] = METADATA(p);
env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + env->me_psize);
return MDB_SUCCESS;
}
int int
mdb_env_set_mapsize(MDB_env *env, size_t size) mdb_env_set_mapsize(MDB_env *env, size_t size)
{ {
if (env->me_map) /* If env is already open, caller is responsible for making
return EINVAL; * sure there are no active txns.
*/
if (env->me_map) {
int rc;
void *old;
if (env->me_txn)
return EINVAL;
if (!size)
size = env->me_metas[mdb_env_pick_meta(env)]->mm_mapsize;
munmap(env->me_map, env->me_mapsize);
env->me_mapsize = size;
old = (env->me_flags & MDB_FIXEDMAP) ? env->me_map : NULL;
rc = mdb_env_map(env, old, 1);
if (rc)
return rc;
}
env->me_mapsize = size; env->me_mapsize = size;
if (env->me_psize) if (env->me_psize)
env->me_maxpg = env->me_mapsize / env->me_psize; env->me_maxpg = env->me_mapsize / env->me_psize;
@ -3282,12 +3368,17 @@ static int
mdb_env_open2(MDB_env *env) mdb_env_open2(MDB_env *env)
{ {
unsigned int flags = env->me_flags; unsigned int flags = env->me_flags;
int i, newenv = 0; int i, newenv = 0, rc;
MDB_meta meta; MDB_meta meta;
MDB_page *p;
#ifndef _WIN32 #ifdef _WIN32
int prot; /* See if we should use QueryLimited */
#endif rc = GetVersion();
if ((rc & 0xff) > 5)
env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
else
env->me_pidquery = PROCESS_QUERY_INFORMATION;
#endif /* _WIN32 */
memset(&meta, 0, sizeof(meta)); memset(&meta, 0, sizeof(meta));
@ -3296,6 +3387,9 @@ mdb_env_open2(MDB_env *env)
return i; return i;
DPUTS("new mdbenv"); DPUTS("new mdbenv");
newenv = 1; newenv = 1;
GET_PAGESIZE(env->me_psize);
} else {
env->me_psize = meta.mm_psize;
} }
/* Was a mapsize configured? */ /* Was a mapsize configured? */
@ -3313,66 +3407,9 @@ mdb_env_open2(MDB_env *env)
env->me_mapsize = minsize; env->me_mapsize = minsize;
} }
#ifdef _WIN32 rc = mdb_env_map(env, meta.mm_address, newenv);
{ if (rc)
int rc; return rc;
HANDLE mh;
LONG sizelo, sizehi;
sizelo = env->me_mapsize & 0xffffffff;
sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */
/* See if we should use QueryLimited */
rc = GetVersion();
if ((rc & 0xff) > 5)
env->me_pidquery = MDB_PROCESS_QUERY_LIMITED_INFORMATION;
else
env->me_pidquery = PROCESS_QUERY_INFORMATION;
/* Windows won't create mappings for zero length files.
* Just allocate the maxsize right now.
*/
if (newenv) {
if (SetFilePointer(env->me_fd, sizelo, &sizehi, 0) != (DWORD)sizelo
|| !SetEndOfFile(env->me_fd)
|| SetFilePointer(env->me_fd, 0, NULL, 0) != 0)
return ErrCode();
}
mh = CreateFileMapping(env->me_fd, NULL, flags & MDB_WRITEMAP ?
PAGE_READWRITE : PAGE_READONLY,
sizehi, sizelo, NULL);
if (!mh)
return ErrCode();
env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ?
FILE_MAP_WRITE : FILE_MAP_READ,
0, 0, env->me_mapsize, meta.mm_address);
rc = env->me_map ? 0 : ErrCode();
CloseHandle(mh);
if (rc)
return rc;
}
#else
i = MAP_SHARED;
prot = PROT_READ;
if (flags & MDB_WRITEMAP) {
prot |= PROT_WRITE;
if (ftruncate(env->me_fd, env->me_mapsize) < 0)
return ErrCode();
}
env->me_map = mmap(meta.mm_address, env->me_mapsize, prot, i,
env->me_fd, 0);
if (env->me_map == MAP_FAILED) {
env->me_map = NULL;
return ErrCode();
}
/* Turn off readahead. It's harmful when the DB is larger than RAM. */
#ifdef MADV_RANDOM
madvise(env->me_map, env->me_mapsize, MADV_RANDOM);
#else
#ifdef POSIX_MADV_RANDOM
posix_madvise(env->me_map, env->me_mapsize, POSIX_MADV_RANDOM);
#endif /* POSIX_MADV_RANDOM */
#endif /* MADV_RANDOM */
#endif /* _WIN32 */
if (newenv) { if (newenv) {
if (flags & MDB_FIXEDMAP) if (flags & MDB_FIXEDMAP)
@ -3381,24 +3418,11 @@ mdb_env_open2(MDB_env *env)
if (i != MDB_SUCCESS) { if (i != MDB_SUCCESS) {
return i; return i;
} }
} else if (meta.mm_address && env->me_map != meta.mm_address) {
/* Can happen because the address argument to mmap() is just a
* hint. mmap() can pick another, e.g. if the range is in use.
* The MAP_FIXED flag would prevent that, but then mmap could
* instead unmap existing pages to make room for the new map.
*/
return EBUSY; /* TODO: Make a new MDB_* error code? */
} }
env->me_psize = meta.mm_psize;
env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1; env->me_maxfree_1pg = (env->me_psize - PAGEHDRSZ) / sizeof(pgno_t) - 1;
env->me_nodemax = (env->me_psize - PAGEHDRSZ) / MDB_MINKEYS; env->me_nodemax = (env->me_psize - PAGEHDRSZ) / MDB_MINKEYS;
env->me_maxpg = env->me_mapsize / env->me_psize; env->me_maxpg = env->me_mapsize / env->me_psize;
p = (MDB_page *)env->me_map;
env->me_metas[0] = METADATA(p);
env->me_metas[1] = (MDB_meta *)((char *)env->me_metas[0] + meta.mm_psize);
#if MDB_DEBUG #if MDB_DEBUG
{ {
int toggle = mdb_env_pick_meta(env); int toggle = mdb_env_pick_meta(env);

Write Preview
Loading…
Cancel
Save