ITS#9364 Add crypto support to all tools

Using dynamically loaded crypto modules

libraries/liblmdb/Makefile

@@ -24,9 +24,11 @@ W	= -W -Wall -Wno-unused-parameter -Wbad-function-cast -Wuninitialized
 THREADS = -pthread
 OPT = -O2 -g
 CFLAGS	= $(THREADS) $(OPT) $(W) $(XCFLAGS)
+LDFLAGS = $(THREADS)
 LDLIBS	= 
 SOLIBS	= 
 SOEXT	= .so
+LDL		= -ldl
 prefix	= /usr/local
 exec_prefix = $(prefix)
 bindir = $(exec_prefix)/bin
@@ -72,11 +74,16 @@ liblmdb$(SOEXT):	mdb.lo midl.lo
 #	$(CC) $(LDFLAGS) -pthread -shared -Wl,-Bsymbolic -o $@ mdb.o midl.o $(SOLIBS)
 	$(CC) $(LDFLAGS) -pthread -shared -o $@ mdb.lo midl.lo $(SOLIBS)
 
-mdb_stat: mdb_stat.o liblmdb.a
+mdb_stat: mdb_stat.o module.o liblmdb.a
-mdb_copy: mdb_copy.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_dump: mdb_dump.o liblmdb.a
+mdb_copy: mdb_copy.o module.o liblmdb.a
-mdb_load: mdb_load.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
-mdb_drop: mdb_drop.o liblmdb.a
+mdb_dump: mdb_dump.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_load: mdb_load.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
+mdb_drop: mdb_drop.o module.o liblmdb.a
+	$(CC) $(LDFLAGS) -o $@ $^ $(LDL)
 mtest:    mtest.o    liblmdb.a
 mtest2:	mtest2.o liblmdb.a
 mtest3:	mtest3.o liblmdb.a
@@ -85,9 +92,11 @@ mtest5:	mtest5.o liblmdb.a
 mtest6:	mtest6.o liblmdb.a
 mtest_remap:  mtest_remap.o liblmdb.a
 mtest_enc:    mtest_enc.o chacha8.o liblmdb.a
-mtest_enc2:	  mtest_enc2.o crypto.o liblmdb.a
+mtest_enc2:	  mtest_enc2.o module.o liblmdb.a crypto.lm
-	$(CC) $(LDFLAGS) -pthread -o $@ $^ -lcrypto
+	$(CC) $(LDFLAGS) -pthread -o $@ mtest_enc2.o module.o liblmdb.a $(LDL)
 
+crypto.lm:	crypto.c
+	$(CC) -shared -o $@ -lcrypto
 
 mdb.o: mdb.c lmdb.h midl.h
 	$(CC) $(CFLAGS) $(CPPFLAGS) -c mdb.c

libraries/liblmdb/crypto.c

@@ -1,3 +1,15 @@
+/* crypto.c - LMDB encryption helper module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
 #include <string.h>
 
 #include <openssl/engine.h>
@@ -8,7 +20,7 @@ MDB_crypto_hooks MDB_crypto;
 
 static EVP_CIPHER *cipher;
 
-static int str2key(const char *passwd, MDB_val *key)
+static int mcf_str2key(const char *passwd, MDB_val *key)
 {
 	unsigned int size;
 	EVP_MD_CTX *mdctx = EVP_MD_CTX_new();
@@ -66,7 +78,7 @@ typedef struct {
     size_t tls_payload_length;
 } EVP_CHACHA_AEAD_CTX;
 
-static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
+static int mcf_encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
 {
 	unsigned char iv[12];
 	int ivl, outl, rc;
@@ -93,17 +105,17 @@ static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int enc
 	return rc == 0;
 }
 
-static const MDB_crypto_funcs table = {
+static const MDB_crypto_funcs mcf_table = {
-	str2key,
+	mcf_str2key,
-	encfunc,
+	mcf_encfunc,
 	NULL,
-	32,
+	CHACHA_KEY_SIZE,
-	16,
+	POLY1305_BLOCK_SIZE,
 	0
 };
 
 MDB_crypto_funcs *MDB_crypto()
 {
 	cipher = (EVP_CIPHER *)EVP_chacha20_poly1305();
-	return (MDB_crypto_funcs *)&table;
+	return (MDB_crypto_funcs *)&mcf_table;
 }

libraries/liblmdb/mdb_copy.c

@@ -21,6 +21,7 @@
 #include <stdlib.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 static void
 sighandle(int sig)
@@ -34,6 +35,9 @@ int main(int argc,char * argv[])
 	const char *progname = argv[0], *act;
 	unsigned flags = MDB_RDONLY;
 	unsigned cpflags = 0;
+	char *module = NULL, *password = NULL;
+	void *mlm = NULL;
+	char *errmsg;
 
 	for (; argc > 1 && argv[1][0] == '-'; argc--, argv++) {
 		if (argv[1][1] == 'n' && argv[1][2] == '\0')
@@ -45,15 +49,24 @@ int main(int argc,char * argv[])
 		else if (argv[1][1] == 'V' && argv[1][2] == '\0') {
 			printf("%s\n", MDB_VERSION_STRING);
 			exit(0);
+		} else if (argv[1][1] == 'm' && argv[1][2] == '\0') {
+			module = argv[2];
+			argc--;
+			argv++;
+		} else if (argv[1][1] == 'w' && argv[1][2] == '\0') {
+			password = argv[2];
+			argc--;
+			argv++;
 		} else
 			argc = 0;
 	}
 
 	if (argc<2 || argc>3) {
-		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] srcpath [dstpath]\n", progname);
+		fprintf(stderr, "usage: %s [-V] [-c] [-n] [-v] [-m module [-w password]] srcpath [dstpath]\n", progname);
 		exit(EXIT_FAILURE);
 	}
 
+
 #ifdef SIGPIPE
 	signal(SIGPIPE, sighandle);
 #endif
@@ -66,6 +79,13 @@ int main(int argc,char * argv[])
 	act = "opening environment";
 	rc = mdb_env_create(&env);
 	if (rc == MDB_SUCCESS) {
+		if (module) {
+			mlm = mlm_setup(env, module, password, &errmsg);
+			if (!mlm) {
+				fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+				exit(EXIT_FAILURE);
+			}
+		}
 		rc = mdb_env_open(env, argv[1], flags, 0600);
 	}
 	if (rc == MDB_SUCCESS) {
@@ -79,6 +99,8 @@ int main(int argc,char * argv[])
 		fprintf(stderr, "%s: %s failed, error %d (%s)\n",
 			progname, act, rc, mdb_strerror(rc));
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }

libraries/liblmdb/mdb_drop.c

@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 static volatile sig_atomic_t gotsig;
 
@@ -29,7 +30,7 @@ static void dumpsig( int sig )
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-d] [-m module [-w password]] [-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -43,6 +44,9 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int envflags = 0, delete = 0;
+	char *module = NULL, *password = NULL;
+	void *mlm = NULL;
+	char *errmsg;
 
 	if (argc < 2) {
 		usage(prog);
@@ -54,7 +58,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) empty the main DB
 	 */
-	while ((i = getopt(argc, argv, "dns:V")) != EOF) {
+	while ((i = getopt(argc, argv, "dm:ns:w:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -69,6 +73,12 @@ int main(int argc, char *argv[])
 		case 's':
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -92,6 +102,13 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 		return EXIT_FAILURE;
 	}
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
 
 	mdb_env_set_maxdbs(env, 2);
 
@@ -130,6 +147,8 @@ txn_abort:
 		mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }

libraries/liblmdb/mdb_dump.c

@@ -19,6 +19,7 @@
 #include <unistd.h>
 #include <signal.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define Yu	MDB_PRIy(u)
 
@@ -153,7 +154,7 @@ static int dumpit(MDB_txn *txn, MDB_dbi dbi, char *name)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-f output] [-l] [-n] [-p] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -167,6 +168,8 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int alldbs = 0, envflags = 0, list = 0;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	if (argc < 2) {
 		usage(prog);
@@ -181,7 +184,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) dump only the main DB
 	 */
-	while ((i = getopt(argc, argv, "af:lnps:vV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:lm:nps:vw:V")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -216,6 +219,12 @@ int main(int argc, char *argv[])
 				usage(prog);
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -240,6 +249,14 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
+
 	if (alldbs || subname) {
 		mdb_env_set_maxdbs(env, 2);
 	}

libraries/liblmdb/mdb_load.c

@@ -18,6 +18,7 @@
 #include <ctype.h>
 #include <unistd.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define PRINT	1
 #define NOHDR	2
@@ -276,7 +277,7 @@ badend:
 
 static void usage(void)
 {
-	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-s name] [-N] [-T] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-a] [-f input] [-n] [-m module [-w password]] [-s name] [-N] [-T] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -296,6 +297,8 @@ int main(int argc, char *argv[])
 	int envflags = MDB_NOSYNC, putflags = 0;
 	int dohdr = 0, append = 0;
 	MDB_val prevk;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	prog = argv[0];
 
@@ -311,7 +314,7 @@ int main(int argc, char *argv[])
 	 * -T: read plaintext
 	 * -V: print version and exit
 	 */
-	while ((i = getopt(argc, argv, "af:ns:NTV")) != EOF) {
+	while ((i = getopt(argc, argv, "af:m:ns:w:NTV")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -339,6 +342,12 @@ int main(int argc, char *argv[])
 		case 'T':
 			mode |= NOHDR | PRINT;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage();
 		}
@@ -359,6 +368,13 @@ int main(int argc, char *argv[])
 		fprintf(stderr, "mdb_env_create failed, error %d %s\n", rc, mdb_strerror(rc));
 		return EXIT_FAILURE;
 	}
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
 
 	mdb_env_set_maxdbs(env, 2);
 
@@ -487,6 +503,8 @@ txn_abort:
 	mdb_txn_abort(txn);
 env_close:
 	mdb_env_close(env);
+	if (mlm)
+		mlm_unload(mlm);
 
 	return rc ? EXIT_FAILURE : EXIT_SUCCESS;
 }

libraries/liblmdb/mdb_stat.c

@@ -16,15 +16,14 @@
 #include <string.h>
 #include <unistd.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define Z	MDB_FMT_Z
 #define Yu	MDB_PRIy(u)
 
 static void prstat(MDB_stat *ms)
 {
-#if 0
 	printf("  Page size: %u\n", ms->ms_psize);
-#endif
 	printf("  Tree depth: %u\n", ms->ms_depth);
 	printf("  Branch pages: %"Yu"\n",   ms->ms_branch_pages);
 	printf("  Leaf pages: %"Yu"\n",     ms->ms_leaf_pages);
@@ -34,7 +33,7 @@ static void prstat(MDB_stat *ms)
 
 static void usage(char *prog)
 {
-	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-a|-s subdb] dbpath\n", prog);
+	fprintf(stderr, "usage: %s [-V] [-n] [-e] [-r[r]] [-f[f[f]]] [-v] [-m module [-w password]] [-a|-s subdb] dbpath\n", prog);
 	exit(EXIT_FAILURE);
 }
 
@@ -50,6 +49,8 @@ int main(int argc, char *argv[])
 	char *envname;
 	char *subname = NULL;
 	int alldbs = 0, envinfo = 0, envflags = 0, freinfo = 0, rdrinfo = 0;
+	char *module = NULL, *password = NULL, *errmsg;
+	void *mlm = NULL;
 
 	if (argc < 2) {
 		usage(prog);
@@ -65,7 +66,7 @@ int main(int argc, char *argv[])
 	 * -V: print version and exit
 	 * (default) print stat of only the main DB
 	 */
-	while ((i = getopt(argc, argv, "Vaefnrs:v")) != EOF) {
+	while ((i = getopt(argc, argv, "Vaefm:nrs:vw:")) != EOF) {
 		switch(i) {
 		case 'V':
 			printf("%s\n", MDB_VERSION_STRING);
@@ -96,6 +97,12 @@ int main(int argc, char *argv[])
 				usage(prog);
 			subname = optarg;
 			break;
+		case 'm':
+			module = optarg;
+			break;
+		case 'w':
+			password = optarg;
+			break;
 		default:
 			usage(prog);
 		}
@@ -111,6 +118,14 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 
+	if (module) {
+		mlm = mlm_setup(env, module, password, &errmsg);
+		if (!mlm) {
+			fprintf(stderr, "Failed to load crypto module: %s\n", errmsg);
+			goto env_close;
+		}
+	}
+
 	if (alldbs || subname) {
 		mdb_env_set_maxdbs(env, 4);
 	}

libraries/liblmdb/module.c

@@ -0,0 +1,101 @@
+/* module.c - helper for dynamically loading crypto module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <dlfcn.h>
+#endif
+
+#include <stddef.h>
+#include <string.h>
+
+#include "lmdb.h"
+#include "module.h"
+
+void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg)
+{
+	MDB_crypto_hooks *hookfunc;
+	void *ret = NULL;
+	if (!name)
+		name = "MDB_crypto";
+
+#ifdef _WIN32
+	{
+		HINSTANCE mlm = LoadLibrary(file);
+		if (mlm) {
+			hookfunc = GetProcAddress(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				FreeLibrary(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = GetLastError();
+		}
+		ret = (void *)mlm;
+	}
+#else
+	{
+		void *mlm = dlopen(file, RTLD_NOW);
+		if (mlm) {
+			hookfunc = dlsym(mlm, name);
+			if (hookfunc)
+				*mcf_ptr = hookfunc();
+			else {
+				*errmsg = "Crypto hook function not found";
+				dlclose(mlm);
+				mlm = NULL;
+			}
+		} else {
+			*errmsg = dlerror();
+		}
+		ret = mlm;
+	}
+#endif
+	return ret;
+}
+
+void mlm_unload(void *mlm)
+{
+#ifdef _WIN32
+	FreeLibrary((HINSTANCE)mlm);
+#else
+	dlclose(mlm);
+#endif
+}
+
+void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg)
+{
+	MDB_crypto_funcs *cf;
+	MDB_val enckey = {0};
+	void *mlm = mlm_load(file, NULL, &cf, errmsg);
+	if (mlm) {
+		if (cf->mcf_sumfunc) {
+			mdb_env_set_checksum(env, cf->mcf_sumfunc, cf->mcf_sumsize);
+		}
+		if (cf->mcf_encfunc && password) {
+			char keybuf[2048];
+			enckey.mv_data = keybuf;
+			enckey.mv_size = cf->mcf_keysize;
+			if (cf->mcf_str2key)
+				cf->mcf_str2key(password, &enckey);
+			else
+				strncpy(enckey.mv_data, password, enckey.mv_size);
+			mdb_env_set_encrypt(env, cf->mcf_encfunc, &enckey, cf->mcf_esumsize);
+			memset(enckey.mv_data, 0, enckey.mv_size);
+		}
+	}
+	return mlm;
+}

libraries/liblmdb/module.h

@@ -0,0 +1,16 @@
+/* module.h - helper for dynamically loading crypto module */
+/*
+ * Copyright 2020 Howard Chu, Symas Corp.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted only as authorized by the Symas
+ * Dual-Use License.
+ *
+ * A copy of this license is available in the file LICENSE in the
+ * source distribution.
+ */
+
+void *mlm_load(const char *file, const char *name, MDB_crypto_funcs **mcf_ptr, char **errmsg);
+void mlm_unload(void *lm);
+void *mlm_setup(MDB_env *env, const char *file, const char *password, char **errmsg);

libraries/liblmdb/mtest_enc2.c

@@ -15,13 +15,13 @@
 #include <stdlib.h>
 #include <time.h>
 #include "lmdb.h"
+#include "module.h"
 
 #define E(expr) CHECK((rc = (expr)) == MDB_SUCCESS, #expr)
 #define RES(err, expr) ((rc = expr) == (err) || (CHECK(!rc, #expr), 0))
 #define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
 	"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
 
-extern MDB_crypto_hooks MDB_crypto;
 MDB_crypto_funcs *cf;
 
 int main(int argc,char * argv[])
@@ -40,6 +40,8 @@ int main(int argc,char * argv[])
 	char sval[32] = "";
 	char password[] = "This is my passphrase for now...";
 	char *ekey;
+	void *lm;
+	char *errmsg;
 
 	srand(time(NULL));
 
@@ -50,7 +52,11 @@ int main(int argc,char * argv[])
 			values[i] = rand()%1024;
 	    }
     
-		cf = MDB_crypto();
+		lm = lm_load("./crypto.lm", NULL, &cf, &errmsg);
+		if (!lm) {
+			fprintf(stderr,"Failed to load crypto module: %s\n", errmsg);
+			exit(1);
+		}
 
 		E(mdb_env_create(&env));
 		E(mdb_env_set_maxreaders(env, 1));
@@ -193,6 +199,7 @@ int main(int argc,char * argv[])
 
 		mdb_dbi_close(env, dbi);
 		mdb_env_close(env);
+		lm_unload(lm);
 
 	return 0;
 }