diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index 9d24aef..a90372f 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -35,15 +35,17 @@ #ifndef _GNU_SOURCE #define _GNU_SOURCE 1 #endif -#include -#include #ifdef _WIN32 +#include #include /** getpid() returns int; MinGW defines pid_t but MinGW64 typedefs it * as int64 which is wrong. MSVC doesn't define it at all, so just * don't use it. */ #define MDB_PID_T int +#define MDB_THR_T DWORD +#include +#include #ifdef __GNUC__ # include #else @@ -55,7 +57,10 @@ # endif #endif #else +#include +#include #define MDB_PID_T pid_t +#define MDB_THR_T pthread_t #include #include #include @@ -169,7 +174,8 @@ #ifdef _WIN32 #define MDB_USE_HASH 1 #define MDB_PIDLOCK 0 -#define pthread_t DWORD +#define THREAD_RET DWORD +#define pthread_t HANDLE #define pthread_mutex_t HANDLE #define pthread_key_t DWORD #define pthread_self() GetCurrentThreadId() @@ -180,6 +186,8 @@ #define pthread_setspecific(x,y) (TlsSetValue(x,y) ? 0 : ErrCode()) #define pthread_mutex_unlock(x) ReleaseMutex(x) #define pthread_mutex_lock(x) WaitForSingleObject(x, INFINITE) +#define THREAD_CREATE(thr,start,arg) thr=CreateThread(NULL,0,start,arg,0,NULL) +#define THREAD_FINISH(thr) WaitForSingleObject(thr, INFINITE) #define LOCK_MUTEX_R(env) pthread_mutex_lock((env)->me_rmutex) #define UNLOCK_MUTEX_R(env) pthread_mutex_unlock((env)->me_rmutex) #define LOCK_MUTEX_W(env) pthread_mutex_lock((env)->me_wmutex) @@ -198,7 +206,9 @@ #endif #define Z "I" #else - +#define THREAD_RET void * +#define THREAD_CREATE(thr,start,arg) pthread_create(&thr,NULL,start,arg) +#define THREAD_FINISH(thr) pthread_join(thr,NULL) #define Z "z" /**< printf format modifier for size_t */ /** For MDB_LOCK_FORMAT: True if readers take a pid lock in the lockfile */ @@ -537,7 +547,7 @@ typedef struct MDB_rxbody { /** The process ID of the process owning this reader txn. */ MDB_PID_T mrb_pid; /** The thread ID of the thread owning this txn. */ - pthread_t mrb_tid; + MDB_THR_T mrb_tid; } MDB_rxbody; /** The actual reader record, with cacheline padding. */ @@ -2387,7 +2397,7 @@ mdb_txn_renew0(MDB_txn *txn) return MDB_BAD_RSLOT; } else { MDB_PID_T pid = env->me_pid; - pthread_t tid = pthread_self(); + MDB_THR_T tid = pthread_self(); if (!env->me_live_reader) { rc = mdb_reader_pid(env, Pidset, pid); @@ -3534,8 +3544,17 @@ mdb_env_map(MDB_env *env, void *addr, int newsize) int rc; HANDLE mh; LONG sizelo, sizehi; - sizelo = env->me_mapsize & 0xffffffff; - sizehi = env->me_mapsize >> 16 >> 16; /* only needed on Win64 */ + size_t msize; + + if (flags & MDB_RDONLY) { + msize = 0; + sizelo = 0; + sizehi = 0; + } else { + msize = env->me_mapsize; + sizelo = msize & 0xffffffff; + sizehi = msize >> 16 >> 16; /* only needed on Win64 */ + } /* Windows won't create mappings for zero length files. * Just allocate the maxsize right now. @@ -3553,7 +3572,7 @@ mdb_env_map(MDB_env *env, void *addr, int newsize) return ErrCode(); env->me_map = MapViewOfFileEx(mh, flags & MDB_WRITEMAP ? FILE_MAP_WRITE : FILE_MAP_READ, - 0, 0, env->me_mapsize, addr); + 0, 0, msize, addr); rc = env->me_map ? 0 : ErrCode(); CloseHandle(mh); if (rc) @@ -8018,7 +8037,6 @@ typedef struct mdb_copy { pthread_mutex_t mc_mutex[2]; char *mc_wbuf[2]; char *mc_over[2]; - void *mc_free; MDB_env *mc_env; MDB_txn *mc_txn; int mc_wlen[2]; @@ -8029,16 +8047,17 @@ typedef struct mdb_copy { int mc_toggle; } mdb_copy; -static void * +static THREAD_RET mdb_env_copythr(void *arg) { mdb_copy *my = arg; char *ptr; - int wsize; - int toggle = 0, len, rc; + int toggle = 0, wsize, rc; #ifdef _WIN32 + DWORD len; #define DO_WRITE(rc, fd, ptr, w2, len) rc = WriteFile(fd, ptr, w2, &len, NULL) #else + int len; #define DO_WRITE(rc, fd, ptr, w2, len) len = write(fd, ptr, w2); rc = (len >= 0) #endif @@ -8083,7 +8102,7 @@ again: } toggle ^= 1; } - return NULL; + return (THREAD_RET)0; #undef DO_WRITE } @@ -8228,10 +8247,6 @@ again: continue; } } - if (mc.mc_top) { - ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); - SETPGNO(ni, my->mc_next_pgno); - } if (my->mc_wlen[toggle] >= MDB_WBUF) { rc = mdb_env_cthr_toggle(my); if (rc) @@ -8242,9 +8257,17 @@ again: mdb_page_copy(mo, mp, my->mc_env->me_psize); mo->mp_pgno = my->mc_next_pgno++; my->mc_wlen[toggle] += my->mc_env->me_psize; - mdb_cursor_pop(&mc); + if (mc.mc_top) { + /* Update parent if there is one */ + ni = NODEPTR(mc.mc_pg[mc.mc_top-1], mc.mc_ki[mc.mc_top-1]); + SETPGNO(ni, mo->mp_pgno); + mdb_cursor_pop(&mc); + } else { + /* Otherwise we're done */ + *pg = mo->mp_pgno; + break; + } } - *pg = mo->mp_pgno; done: free(buf); return rc; @@ -8260,13 +8283,20 @@ mdb_env_copyfd2(MDB_env *env, HANDLE fd) pthread_t thr; int rc; - rc = posix_memalign(&my.mc_free, env->me_psize, MDB_WBUF*2); - if (rc) - return rc; - my.mc_wbuf[0] = my.mc_free; - my.mc_wbuf[1] = my.mc_free + MDB_WBUF; +#ifdef _WIN32 + my.mc_mutex[0] = CreateMutex(NULL, FALSE, NULL); + my.mc_mutex[1] = CreateMutex(NULL, FALSE, NULL); + my.mc_wbuf[0] = _aligned_malloc(MDB_WBUF*2, env->me_psize); + if (my.mc_wbuf[0] == NULL) + return errno; +#else pthread_mutex_init(&my.mc_mutex[0], NULL); pthread_mutex_init(&my.mc_mutex[1], NULL); + rc = posix_memalign((void **)&my.mc_wbuf[0], env->me_psize, MDB_WBUF*2); + if (rc) + return rc; +#endif + my.mc_wbuf[1] = my.mc_wbuf[0] + MDB_WBUF; my.mc_wlen[0] = 0; my.mc_wlen[1] = 0; my.mc_olen[0] = 0; @@ -8335,16 +8365,24 @@ mdb_env_copyfd2(MDB_env *env, HANDLE fd) } my.mc_wlen[0] = env->me_psize * 2; my.mc_txn = txn; - pthread_create(&thr, NULL, mdb_env_copythr, &my); + THREAD_CREATE(thr, mdb_env_copythr, &my); rc = mdb_env_cwalk(&my, &txn->mt_dbs[1].md_root, 0); if (rc == MDB_SUCCESS && my.mc_wlen[my.mc_toggle]) rc = mdb_env_cthr_toggle(&my); my.mc_wlen[my.mc_toggle] = 0; pthread_mutex_unlock(&my.mc_mutex[my.mc_toggle]); - pthread_join(thr, NULL); + THREAD_FINISH(thr); leave: mdb_txn_abort(txn); - free(my.mc_free); +#ifdef _WIN32 + CloseHandle(my.mc_mutex[1]); + CloseHandle(my.mc_mutex[0]); + _aligned_free(my.mc_wbuf[0]); +#else + pthread_mutex_destroy(&my.mc_mutex[1]); + pthread_mutex_destroy(&my.mc_mutex[0]); + free(my.mc_wbuf[0]); +#endif return rc; } diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c index 9255325..3b01f96 100644 --- a/libraries/liblmdb/mdb_dump.c +++ b/libraries/liblmdb/mdb_dump.c @@ -20,6 +20,12 @@ #include #include "lmdb.h" +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + #define PRINT 1 static int mode; @@ -109,7 +115,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) if (name) printf("database=%s\n", name); printf("type=btree\n"); - printf("mapsize=%zu\n", info.me_mapsize); + printf("mapsize=%" Z "u\n", info.me_mapsize); if (info.me_mapaddr) printf("mapaddr=%p\n", info.me_mapaddr); printf("maxreaders=%u\n", info.me_maxreaders); diff --git a/libraries/liblmdb/mdb_load.c b/libraries/liblmdb/mdb_load.c index ec9f9ad..17f4757 100644 --- a/libraries/liblmdb/mdb_load.c +++ b/libraries/liblmdb/mdb_load.c @@ -32,12 +32,18 @@ static int flags; static char *prog; -static int eof; +static int Eof; static MDB_envinfo info; static MDB_val kbuf, dbuf; +#ifdef _WIN32 +#define Z "I" +#else +#define Z "z" +#endif + #define STRLENOF(s) (sizeof(s)-1) typedef struct flagbit { @@ -69,7 +75,7 @@ static void readhdr() if (!strncmp(dbuf.mv_data, "VERSION=", STRLENOF("VERSION="))) { version=atoi((char *)dbuf.mv_data+STRLENOF("VERSION=")); if (version > 3) { - fprintf(stderr, "%s: line %zd: unsupported VERSION %d\n", + fprintf(stderr, "%s: line %" Z "d: unsupported VERSION %d\n", prog, lineno, version); exit(EXIT_FAILURE); } @@ -79,7 +85,7 @@ static void readhdr() if (!strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "print", STRLENOF("print"))) mode |= PRINT; else if (strncmp((char *)dbuf.mv_data+STRLENOF("FORMAT="), "bytevalue", STRLENOF("bytevalue"))) { - fprintf(stderr, "%s: line %zd: unsupported FORMAT %s\n", + fprintf(stderr, "%s: line %" Z "d: unsupported FORMAT %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("FORMAT=")); exit(EXIT_FAILURE); } @@ -90,7 +96,7 @@ static void readhdr() subname = strdup((char *)dbuf.mv_data+STRLENOF("database=")); } else if (!strncmp(dbuf.mv_data, "type=", STRLENOF("type="))) { if (strncmp((char *)dbuf.mv_data+STRLENOF("type="), "btree", STRLENOF("btree"))) { - fprintf(stderr, "%s: line %zd: unsupported type %s\n", + fprintf(stderr, "%s: line %" Z "d: unsupported type %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("type=")); exit(EXIT_FAILURE); } @@ -100,7 +106,7 @@ static void readhdr() if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("mapaddr="), "%p", &info.me_mapaddr); if (i != 1) { - fprintf(stderr, "%s: line %zd: invalid mapaddr %s\n", + fprintf(stderr, "%s: line %" Z "d: invalid mapaddr %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapaddr=")); exit(EXIT_FAILURE); } @@ -108,9 +114,9 @@ static void readhdr() int i; ptr = memchr(dbuf.mv_data, '\n', dbuf.mv_size); if (ptr) *ptr = '\0'; - i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%zu", &info.me_mapsize); + i = sscanf((char *)dbuf.mv_data+STRLENOF("mapsize="), "%" Z "u", &info.me_mapsize); if (i != 1) { - fprintf(stderr, "%s: line %zd: invalid mapsize %s\n", + fprintf(stderr, "%s: line %" Z "d: invalid mapsize %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("mapsize=")); exit(EXIT_FAILURE); } @@ -120,7 +126,7 @@ static void readhdr() if (ptr) *ptr = '\0'; i = sscanf((char *)dbuf.mv_data+STRLENOF("maxreaders="), "%u", &info.me_maxreaders); if (i != 1) { - fprintf(stderr, "%s: line %zd: invalid maxreaders %s\n", + fprintf(stderr, "%s: line %" Z "d: invalid maxreaders %s\n", prog, lineno, (char *)dbuf.mv_data+STRLENOF("maxreaders=")); exit(EXIT_FAILURE); } @@ -136,12 +142,12 @@ static void readhdr() if (!dbflags[i].bit) { ptr = memchr(dbuf.mv_data, '=', dbuf.mv_size); if (!ptr) { - fprintf(stderr, "%s: line %zd: unexpected format\n", + fprintf(stderr, "%s: line %" Z "d: unexpected format\n", prog, lineno); exit(EXIT_FAILURE); } else { *ptr = '\0'; - fprintf(stderr, "%s: line %zd: unrecognized keyword ignored: %s\n", + fprintf(stderr, "%s: line %" Z "d: unrecognized keyword ignored: %s\n", prog, lineno, (char *)dbuf.mv_data); } } @@ -151,7 +157,7 @@ static void readhdr() static void badend() { - fprintf(stderr, "%s: line %zd: unexpected end of input\n", + fprintf(stderr, "%s: line %" Z "d: unexpected end of input\n", prog, lineno); } @@ -178,14 +184,14 @@ static int readline(MDB_val *out, MDB_val *buf) if (!(mode & NOHDR)) { c = fgetc(stdin); if (c == EOF) { - eof = 1; + Eof = 1; return EOF; } if (c != ' ') { lineno++; if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { badend: - eof = 1; + Eof = 1; badend(); return EOF; } @@ -195,7 +201,7 @@ badend: } } if (fgets(buf->mv_data, buf->mv_size, stdin) == NULL) { - eof = 1; + Eof = 1; return EOF; } lineno++; @@ -207,15 +213,15 @@ badend: while (c1[len-1] != '\n') { buf->mv_data = realloc(buf->mv_data, buf->mv_size*2); if (!buf->mv_data) { - eof = 1; - fprintf(stderr, "%s: line %zd: out of memory, line too long\n", + Eof = 1; + fprintf(stderr, "%s: line %" Z "d: out of memory, line too long\n", prog, lineno); return EOF; } c1 = buf->mv_data; c1 += buf->mv_size; if (fgets((char *)c1, buf->mv_size, stdin) == NULL) { - eof = 1; + Eof = 1; badend(); return EOF; } @@ -234,7 +240,7 @@ badend: c1++; c2 += 2; } else { if (c2+3 >= end || !isxdigit(c2[1]) || !isxdigit(c2[2])) { - eof = 1; + Eof = 1; badend(); return EOF; } @@ -248,13 +254,13 @@ badend: } else { /* odd length not allowed */ if (len & 1) { - eof = 1; + Eof = 1; badend(); return EOF; } while (c2 < end) { if (!isxdigit(*c2) || !isxdigit(c2[1])) { - eof = 1; + Eof = 1; badend(); return EOF; } @@ -360,7 +366,7 @@ int main(int argc, char *argv[]) kbuf.mv_size = mdb_env_get_maxkeysize(env) * 2 + 2; kbuf.mv_data = malloc(kbuf.mv_size); - while(!eof) { + while(!Eof) { MDB_val key, data; int batch = 0; flags = 0; @@ -408,7 +414,7 @@ int main(int argc, char *argv[]) if (batch == 100) { rc = mdb_txn_commit(txn); if (rc) { - fprintf(stderr, "%s: line %zd: txn_commit: %s\n", + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; } @@ -428,7 +434,7 @@ int main(int argc, char *argv[]) rc = mdb_txn_commit(txn); txn = NULL; if (rc) { - fprintf(stderr, "%s: line %zd: txn_commit: %s\n", + fprintf(stderr, "%s: line %" Z "d: txn_commit: %s\n", prog, lineno, mdb_strerror(rc)); goto env_close; }