From 3e83b5b27bcd76a5ae086573516f9b76bdc6e2da Mon Sep 17 00:00:00 2001 From: Howard Chu Date: Fri, 20 Jun 2014 05:33:31 -0700 Subject: [PATCH] Add mdb_dump, update copyrights --- libraries/liblmdb/Makefile | 5 +- libraries/liblmdb/lmdb.h | 6 +- libraries/liblmdb/mdb.c | 2 +- libraries/liblmdb/mdb_copy.1 | 10 +- libraries/liblmdb/mdb_copy.c | 7 +- libraries/liblmdb/mdb_dump.1 | 70 +++++++++ libraries/liblmdb/mdb_dump.c | 276 +++++++++++++++++++++++++++++++++++ libraries/liblmdb/mdb_stat.1 | 9 +- libraries/liblmdb/mdb_stat.c | 9 +- libraries/liblmdb/midl.c | 2 +- libraries/liblmdb/midl.h | 2 +- 11 files changed, 382 insertions(+), 16 deletions(-) create mode 100644 libraries/liblmdb/mdb_dump.1 create mode 100644 libraries/liblmdb/mdb_dump.c diff --git a/libraries/liblmdb/Makefile b/libraries/liblmdb/Makefile index 9160c57..a34a9de 100644 --- a/libraries/liblmdb/Makefile +++ b/libraries/liblmdb/Makefile @@ -29,8 +29,8 @@ prefix = /usr/local IHDRS = lmdb.h ILIBS = liblmdb.a liblmdb.so -IPROGS = mdb_stat mdb_copy -IDOCS = mdb_stat.1 mdb_copy.1 +IPROGS = mdb_stat mdb_copy mdb_dump +IDOCS = mdb_stat.1 mdb_copy.1 mdb_dump.1 PROGS = $(IPROGS) mtest mtest2 mtest3 mtest4 mtest5 all: $(ILIBS) $(PROGS) @@ -56,6 +56,7 @@ liblmdb.so: mdb.o midl.o mdb_stat: mdb_stat.o liblmdb.a mdb_copy: mdb_copy.o liblmdb.a +mdb_dump: mdb_dump.o liblmdb.a mtest: mtest.o liblmdb.a mtest2: mtest2.o liblmdb.a mtest3: mtest3.o liblmdb.a diff --git a/libraries/liblmdb/lmdb.h b/libraries/liblmdb/lmdb.h index 5b27973..c7a2590 100644 --- a/libraries/liblmdb/lmdb.h +++ b/libraries/liblmdb/lmdb.h @@ -184,7 +184,7 @@ typedef int mdb_filehandle_t; /** Library minor version */ #define MDB_VERSION_MINOR 9 /** Library patch version */ -#define MDB_VERSION_PATCH 13 +#define MDB_VERSION_PATCH 14 /** Combine args a,b,c into a single integer for easy version comparisons */ #define MDB_VERINT(a,b,c) (((a) << 24) | ((b) << 16) | (c)) @@ -194,10 +194,10 @@ typedef int mdb_filehandle_t; MDB_VERINT(MDB_VERSION_MAJOR,MDB_VERSION_MINOR,MDB_VERSION_PATCH) /** The release date of this library version */ -#define MDB_VERSION_DATE "June 13, 2014" +#define MDB_VERSION_DATE "June 20, 2014" /** A stringifier for the version info */ -#define MDB_VERSTR(a,b,c,d) "MDB " #a "." #b "." #c ": (" d ")" +#define MDB_VERSTR(a,b,c,d) "LMDB " #a "." #b "." #c ": (" d ")" /** A helper for the stringifier macro */ #define MDB_VERFOO(a,b,c,d) MDB_VERSTR(a,b,c,d) diff --git a/libraries/liblmdb/mdb.c b/libraries/liblmdb/mdb.c index fdc0386..2262901 100644 --- a/libraries/liblmdb/mdb.c +++ b/libraries/liblmdb/mdb.c @@ -5,7 +5,7 @@ * BerkeleyDB API, but much simplified. */ /* - * Copyright 2011-2013 Howard Chu, Symas Corp. + * Copyright 2011-2014 Howard Chu, Symas Corp. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/liblmdb/mdb_copy.1 b/libraries/liblmdb/mdb_copy.1 index b4ab753..58c6c5b 100644 --- a/libraries/liblmdb/mdb_copy.1 +++ b/libraries/liblmdb/mdb_copy.1 @@ -1,11 +1,13 @@ -.TH MDB_COPY 1 "2012/12/12" "LMDB 0.9.5" -.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_COPY 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_copy \- LMDB environment copy tool .SH SYNOPSIS .B mdb_copy [\c +.BR \-V ] +[\c .BR \-n ] .B srcpath [\c @@ -24,6 +26,10 @@ for storing the backup. Otherwise, the backup will be written to stdout. .SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP .BR \-n Open LDMB environment(s) which do not use subdirectories. diff --git a/libraries/liblmdb/mdb_copy.c b/libraries/liblmdb/mdb_copy.c index bbf0dc9..87525c0 100644 --- a/libraries/liblmdb/mdb_copy.c +++ b/libraries/liblmdb/mdb_copy.c @@ -37,12 +37,15 @@ int main(int argc,char * argv[]) for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) { if (argv[1][1] == 'n' && argv[1][2] == '\0') flags |= MDB_NOSUBDIR; - else + else if (argv[1][1] == 'V' && argv[1][2] == '\0') { + printf("%s\n", MDB_VERSION_STRING); + exit(0); + } else argc = 0; } if (argc<2 || argc>3) { - fprintf(stderr, "usage: %s [-n] srcpath [dstpath]\n", progname); + fprintf(stderr, "usage: %s [-V] [-n] srcpath [dstpath]\n", progname); exit(EXIT_FAILURE); } diff --git a/libraries/liblmdb/mdb_dump.1 b/libraries/liblmdb/mdb_dump.1 new file mode 100644 index 0000000..a89dd52 --- /dev/null +++ b/libraries/liblmdb/mdb_dump.1 @@ -0,0 +1,70 @@ +.TH MDB_DUMP 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2014 Howard Chu, Symas Corp. All Rights Reserved. +.\" Copying restrictions apply. See COPYRIGHT/LICENSE. +.SH NAME +mdb_dump \- LMDB environment export tool +.SH SYNOPSIS +.B mdb_dump +.BR \ envpath +[\c +.BR \-V ] +[\c +.BI \-f \ file\fR] +[\c +.BR \-l ] +[\c +.BR \-n ] +[\c +.BR \-p ] +[\c +.BR \-a \ | +.BI \-s \ subdb\fR] +.SH DESCRIPTION +The +.B mdb_dump +utility reads a database and writes its contents to the +standard output using a portable flat-text format +understood by the +.BR mdb_load (1) +utility. +.SH OPTIONS +.TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP +.BR \-f \ file +Write to the specified file instead of to the standard output. +.TP +.BR \-l +List the databases stored in the environment. Just the +names will be listed, no data will be output. +.TP +.BR \-n +Dump an LMDB database which does not use subdirectories. +.TP +.BR \-p +If characters in either the key or data items are printing characters (as defined by isprint(3)), output them directly. This option permits users to use standard text editors and tools to modify the contents of databases. + +Note: different systems may have different notions about what characters are considered printing characters, and databases dumped in this manner may be less portable to external systems. +.TP +.BR \-a +Dump all of the subdatabases in the environment. +.TP +.BR \-s \ subdb +Dump a specific subdatabase. If no database is specified, only the main database is dumped. +.SH DIAGNOSTICS +Exit status is zero if no errors occur. +Errors result in a non-zero exit status and +a diagnostic message being written to standard error. + +Dumping and reloading databases that use user-defined comparison functions will result in new databases that use the +default comparison functions. \fB In this case it is quite likely that the reloaded database will be damaged beyond +repair permitting neither record storage nor retrieval.\fP + +The only available workaround is to modify the source for the +.BR mdb_load (1) +utility to load the database using the correct comparison functions. +.SH "SEE ALSO" +.BR mdb_load (1) +.SH AUTHOR +Howard Chu of Symas Corporation diff --git a/libraries/liblmdb/mdb_dump.c b/libraries/liblmdb/mdb_dump.c new file mode 100644 index 0000000..5242519 --- /dev/null +++ b/libraries/liblmdb/mdb_dump.c @@ -0,0 +1,276 @@ +/* mdb_dump.c - memory-mapped database dump tool */ +/* + * Copyright 2011-2014 Howard Chu, Symas Corp. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted only as authorized by the OpenLDAP + * Public License. + * + * A copy of this license is available in the file LICENSE in the + * top-level directory of the distribution or, alternatively, at + * . + */ +#include +#include +#include +#include +#include +#include +#include "lmdb.h" + +#define PRINT 1 +static int mode; + +typedef struct flagbit { + int bit; + char *name; +} flagbit; + +flagbit dbflags[] = { + { MDB_REVERSEKEY, "reversekey" }, + { MDB_DUPSORT, "dupsort" }, + { MDB_INTEGERKEY, "integerkey" }, + { MDB_DUPFIXED, "dupfixed" }, + { MDB_INTEGERDUP, "integerdup" }, + { MDB_REVERSEDUP, "reversedup" }, + { 0, NULL } +}; + +static const char hexc[] = "0123456789abcdef"; + +static void hex(unsigned char c) +{ + putchar(hexc[c >> 4]); + putchar(hexc[c & 0xf]); +} + +static void text(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + if (isprint(*c)) { + putchar(*c); + } else { + putchar('\\'); + hex(*c); + } + c++; + } + putchar('\n'); +} + +static void byte(MDB_val *v) +{ + unsigned char *c, *end; + + putchar(' '); + c = v->mv_data; + end = c + v->mv_size; + while (c < end) { + hex(*c++); + } + putchar('\n'); +} + +/* Dump in BDB-compatible format */ +static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name) +{ + MDB_cursor *mc; + MDB_stat ms; + MDB_val key, data; + unsigned int flags; + int rc, i; + + rc = mdb_dbi_flags(txn, dbi, &flags); + if (rc) return rc; + + rc = mdb_stat(txn, dbi, &ms); + if (rc) return rc; + + printf("VERSION=3\n"); + printf("format=%s\n", mode & PRINT ? "print" : "bytevalue"); + if (name) + printf("database=%s\n", name); + printf("type=btree\n"); + + if (flags & MDB_DUPSORT) + printf("duplicates=1\n"); + + for (i=0; dbflags[i].bit; i++) + if (flags & dbflags[i].bit) + printf("%s=1\n", dbflags[i].name); + + printf("db_pagesize=%d\n", ms.ms_psize); + printf("HEADER=END\n"); + + rc = mdb_cursor_open(txn, dbi, &mc); + if (rc) return rc; + + while ((rc = mdb_cursor_get(mc, &key, &data, MDB_NEXT) == MDB_SUCCESS)) { + if (mode & PRINT) { + text(&key); + text(&data); + } else { + byte(&key); + byte(&data); + } + } + printf("DATA=END\n"); + if (rc == MDB_NOTFOUND) + rc = MDB_SUCCESS; + + return rc; +} + +static void usage(char *prog) +{ + fprintf(stderr, "usage: %s dbpath [-V] [-f output] [-l] [-n] [-p] [-a|-s subdb]\n", prog); + exit(EXIT_FAILURE); +} + +int main(int argc, char *argv[]) +{ + int i, rc; + MDB_env *env; + MDB_txn *txn; + MDB_dbi dbi; + char *prog = argv[0]; + char *envname; + char *subname = NULL; + int alldbs = 0, envflags = 0, list = 0; + + if (argc < 2) { + usage(prog); + } + + /* -a: dump main DB and all subDBs + * -s: dump only the named subDB + * -n: use NOSUBDIR flag on env_open + * -p: use printable characters + * -f: write to file instead of stdout + * -V: print version and exit + * (default) dump only the main DB + */ + while ((i = getopt(argc, argv, "af:lnps:V")) != EOF) { + switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; + case 'l': + list = 1; + /*FALLTHROUGH*/; + case 'a': + if (subname) + usage(prog); + alldbs++; + break; + case 'f': + if (freopen(optarg, "w", stdout) == NULL) { + fprintf(stderr, "%s: %s: reopen: %s\n", + prog, optarg, strerror(errno)); + exit(EXIT_FAILURE); + } + break; + case 'n': + envflags |= MDB_NOSUBDIR; + break; + case 'p': + mode |= PRINT; + break; + case 's': + if (alldbs) + usage(prog); + subname = optarg; + break; + default: + usage(prog); + } + } + + if (optind != argc - 1) + usage(prog); + + envname = argv[optind]; + rc = mdb_env_create(&env); + + if (alldbs || subname) { + mdb_env_set_maxdbs(env, 2); + } + + rc = mdb_env_open(env, envname, envflags | MDB_RDONLY, 0664); + if (rc) { + printf("mdb_env_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_txn_begin(env, NULL, MDB_RDONLY, &txn); + if (rc) { + printf("mdb_txn_begin failed, error %d %s\n", rc, mdb_strerror(rc)); + goto env_close; + } + + rc = mdb_open(txn, subname, 0, &dbi); + if (rc) { + printf("mdb_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + + if (alldbs) { + MDB_cursor *cursor; + MDB_val key; + int count = 0; + + rc = mdb_cursor_open(txn, dbi, &cursor); + if (rc) { + printf("mdb_cursor_open failed, error %d %s\n", rc, mdb_strerror(rc)); + goto txn_abort; + } + while ((rc = mdb_cursor_get(cursor, &key, NULL, MDB_NEXT_NODUP)) == 0) { + char *str; + MDB_dbi db2; + if (memchr(key.mv_data, '\0', key.mv_size)) + continue; + count++; + str = malloc(key.mv_size+1); + memcpy(str, key.mv_data, key.mv_size); + str[key.mv_size] = '\0'; + rc = mdb_open(txn, str, 0, &db2); + if (rc == MDB_SUCCESS) { + if (list) { + printf("%s\n", str); + list++; + } else { + rc = dumpit(txn, db2, str); + } + mdb_close(env, db2); + } + free(str); + if (rc) continue; + } + mdb_cursor_close(cursor); + if (!count) { + fprintf(stderr, "%s: %s does not contain multiple databases\n", prog, envname); + rc = MDB_NOTFOUND; + } else if (rc == MDB_NOTFOUND) { + rc = MDB_SUCCESS; + } + } else { + rc = dumpit(txn, dbi, subname); + } + if (rc && rc != MDB_NOTFOUND) + fprintf(stderr, "%s: %s: %s\n", prog, envname, mdb_strerror(rc)); + + mdb_close(env, dbi); +txn_abort: + mdb_txn_abort(txn); +env_close: + mdb_env_close(env); + + return rc ? EXIT_FAILURE : EXIT_SUCCESS; +} diff --git a/libraries/liblmdb/mdb_stat.1 b/libraries/liblmdb/mdb_stat.1 index 3622772..3d8d461 100644 --- a/libraries/liblmdb/mdb_stat.1 +++ b/libraries/liblmdb/mdb_stat.1 @@ -1,5 +1,5 @@ -.TH MDB_STAT 1 "2012/12/12" "LMDB 0.9.5" -.\" Copyright 2012 Howard Chu, Symas Corp. All Rights Reserved. +.TH MDB_STAT 1 "2014/06/20" "LMDB 0.9.14" +.\" Copyright 2012-2014 Howard Chu, Symas Corp. All Rights Reserved. .\" Copying restrictions apply. See COPYRIGHT/LICENSE. .SH NAME mdb_stat \- LMDB environment status tool @@ -7,6 +7,8 @@ mdb_stat \- LMDB environment status tool .B mdb_stat .BR \ envpath [\c +.BR \-V ] +[\c .BR \-e ] [\c .BR \-f [ f [ f ]]] @@ -23,6 +25,9 @@ The utility displays the status of an LMDB environment. .SH OPTIONS .TP +.BR \-V +Write the library version number to the standard output, and exit. +.TP .BR \-e Display information about the database environment. .TP diff --git a/libraries/liblmdb/mdb_stat.c b/libraries/liblmdb/mdb_stat.c index 40bd4cc..eac2c60 100644 --- a/libraries/liblmdb/mdb_stat.c +++ b/libraries/liblmdb/mdb_stat.c @@ -37,7 +37,7 @@ static void prstat(MDB_stat *ms) static void usage(char *prog) { - fprintf(stderr, "usage: %s dbpath [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); + fprintf(stderr, "usage: %s dbpath [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-a|-s subdb]\n", prog); exit(EXIT_FAILURE); } @@ -64,10 +64,15 @@ int main(int argc, char *argv[]) * -f: print freelist info * -r: print reader info * -n: use NOSUBDIR flag on env_open + * -V: print version and exit * (default) print stat of only the main DB */ - while ((i = getopt(argc, argv, "aefnrs:")) != EOF) { + while ((i = getopt(argc, argv, "Vaefnrs:")) != EOF) { switch(i) { + case 'V': + printf("%s\n", MDB_VERSION_STRING); + exit(0); + break; case 'a': if (subname) usage(prog); diff --git a/libraries/liblmdb/midl.c b/libraries/liblmdb/midl.c index efcaad4..b39d463 100644 --- a/libraries/liblmdb/midl.c +++ b/libraries/liblmdb/midl.c @@ -3,7 +3,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2013 The OpenLDAP Foundation. + * Copyright 2000-2014 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without diff --git a/libraries/liblmdb/midl.h b/libraries/liblmdb/midl.h index 635cd29..f2bb433 100644 --- a/libraries/liblmdb/midl.h +++ b/libraries/liblmdb/midl.h @@ -11,7 +11,7 @@ /* $OpenLDAP$ */ /* This work is part of OpenLDAP Software . * - * Copyright 2000-2013 The OpenLDAP Foundation. + * Copyright 2000-2014 The OpenLDAP Foundation. * All rights reserved. * * Redistribution and use in source and binary forms, with or without