NextGraph
/
lmdb
mirror of https://github.com/nextgraph-org/lmdb.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

ITS#9364 rework crypto API

Browse Source 
And add support for per-page checksums. Reserve space for checksum
at tail of page. Pass pgno+txnid as IV input for encryption.
mdb.master3
Howard Chu 4 years ago
parent d1814f7e5d
commit 8dc526c54f
3 changed files with 232 additions and 75 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 43
      libraries/liblmdb/lmdb.h
  2. 250
      libraries/liblmdb/mdb.c
  3. 14
      libraries/liblmdb/mtest_enc.c

43
libraries/liblmdb/lmdb.h
Unescape View File

@ -317,14 +317,26 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
*
* Encrypt or decrypt the data in src and store the result in dst using the
* provided key. The result must be the same number of bytes as the input.
* The input size will always be a multiple of the page size.
* @param[in] src The input data to be transformed.
* @param[out] dst Storage for the result.
* @param[in] key An array of two values: key[0] is the encryption key,
* and key[1] is the initialization vector.
* @param[in] key An array of three values: key[0] is the encryption key,
* key[1] is the initialization vector, and key[2] is the authentication
* data, if any.
* @param[in] encdec 1 to encrypt, 0 to decrypt.
* @return A non-zero error value on failure and 0 on success.
*/
typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
typedef int (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
/** @brief A callback function used to checksum pages in the env.
*
* Compute the checksum of the data in src and store the result in dst,
* An optional key may be used with keyed hash algorithms.
* @param[in] src The input data to be transformed.
* @param[out] dst Storage for the result.
* @param[in] key An encryption key, if encryption was configured. This
* parameter will be NULL if there is no key.
*/
typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key);
#endif
/** @defgroup mdb_env Environment Flags
@ -506,8 +518,12 @@ typedef enum MDB_cursor_op {
#define MDB_BAD_DBI (-30780)
/** Unexpected problem - txn should abort */
#define MDB_PROBLEM (-30779)
/** Page checksum incorrect */
#define MDB_BAD_CHECKSUM (-30778)
/** Encryption/decryption failed */
#define MDB_CRYPTO_FAIL (-30777)
/** The last defined error code */
#define MDB_LAST_ERRCODE MDB_PROBLEM
#define MDB_LAST_ERRCODE MDB_CRYPTO_FAIL
/** @} */
/** @brief Statistics for a database in the environment */
@ -1017,11 +1033,22 @@ int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
* It implicitly sets #MDB_REMAP_CHUNKS on the env.
* @param[in] env An environment handle returned by #mdb_env_create().
* @param[in] func An #MDB_enc_func function.
* @param[in] key An array of two values: key[0] is the encryption key,
* and key[1] is the initialization vector.
* @param[in] key The encryption key.
* @param[in] size The size of authentication data in bytes, if any.
* Set this to zero for unauthenticated encryption mechanisms.
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size);
/** @brief Set checksums on an environment.
*
* This must be called before #mdb_env_open().
* @param[in] env An environment handle returned by #mdb_env_create().
* @param[in] func An #MDB_sum_func function.
* @param[in] size The size of computed checksum values, in bytes.
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key);
int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
#endif
/** @brief Create a transaction for use with the environment.

250
libraries/liblmdb/mdb.c
Unescape View File

@ -1048,7 +1048,7 @@ typedef struct MDB_page {
#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ))
/** ITS#7713, change PAGEBASE to handle 65536 byte pages */
#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0)
#define PAGEBASE PAGEHDRSZ
/** Number of nodes on a page */
#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1)
@ -1649,11 +1649,15 @@ struct MDB_env {
#if MDB_RPAGE_CACHE
MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */
pthread_mutex_t me_rpmutex; /**< control access to #me_rpages */
MDB_sum_func *me_sumfunc; /**< checksum env data */
unsigned short me_sumsize; /**< size of per-page checksums */
#define MDB_ERPAGE_SIZE 16384
#define MDB_ERPAGE_MAX (MDB_ERPAGE_SIZE-1)
unsigned short me_esumsize; /**< size of per-page authentication data */
unsigned int me_rpcheck;
MDB_enc_func *me_encfunc; /**< encrypt env data */
MDB_val me_enckey[2]; /**< key and IV for env encryption */
MDB_val me_enckey; /**< key for env encryption */
#endif
void *me_userctx; /**< User-settable context */
MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@ -3911,6 +3915,13 @@ mdb_freelist_save(MDB_txn *txn)
return rc;
}
#if MDB_RPAGE_CACHE
static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
static int mdb_page_encrypt(MDB_env *env, MDB_page *in, MDB_page *out, size_t size);
static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size);
static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size);
#endif
/** Flush (some) dirty pages to the map, after clearing their dirty flag.
* @param[in] txn the transaction that's being committed
* @param[in] keep number of initial pages in dirty_list to keep dirty.
@ -3927,9 +3938,6 @@ mdb_page_flush(MDB_txn *txn, int keep)
MDB_OFF_T pos = 0;
pgno_t pgno = 0;
MDB_page *dp = NULL;
#if MDB_RPAGE_CACHE
MDB_page *encp;
#endif
#ifdef _WIN32
OVERLAPPED *ov, *this_ov;
MDB_page *wdp;
@ -4090,16 +4098,17 @@ retry_seek:
wsize = 0;
}
#if MDB_RPAGE_CACHE
if (env->me_sumfunc) {
mdb_page_set_checksum(env, dp, size);
}
if (env->me_encfunc) {
MDB_val in, out;
encp = mdb_page_malloc(txn, nump, 0);
MDB_page *encp = mdb_page_malloc(txn, nump, 0);
if (!encp)
return ENOMEM;
in.mv_size = size;
in.mv_data = dp;
out.mv_size = size;
out.mv_data = encp;
env->me_encfunc(&in, &out, env->me_enckey, 1);
if (mdb_page_encrypt(env, dp, encp, size)) {
mdb_dpage_free_n(env, encp, nump);
return MDB_CRYPTO_FAIL;
}
mdb_dpage_free_n(env, dp, nump);
dp = encp;
dl[i].mptr = dp;
@ -4580,12 +4589,11 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
*(MDB_meta *)METADATA(q) = *meta;
#if MDB_RPAGE_CACHE
if ((env->me_flags & MDB_ENCRYPT) && env->me_enckey[1].mv_size) {
/* save the IV in tail of page 0 */
if (env->me_sumsize) {
/* save the checksum size in tail of page 0 */
char *ptr = (char *)q;
unsigned short *u = (unsigned short *)(ptr-2);
*u = env->me_enckey[1].mv_size;
memcpy(ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_data, env->me_enckey[1].mv_size);
*u = env->me_sumsize;
}
#endif
DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
@ -5331,17 +5339,12 @@ mdb_env_open2(MDB_env *env, int prev)
return MDB_INCOMPATIBLE;
#if MDB_RPAGE_CACHE
if (!newenv && env->me_flags & MDB_ENCRYPT) {
/* for encrypted env, read IV from tail of page 0 */
char *ptr = env->me_map + env->me_psize, *ekey;
if (!newenv && env->me_sumfunc) {
/* for checksums, check sum size from tail of page 0 */
char *ptr = env->me_map + env->me_psize;
unsigned short *u = (unsigned short *)(ptr - 2);
env->me_enckey[1].mv_size = *u;
ekey = realloc(env->me_enckey[0].mv_data, env->me_enckey[0].mv_size + env->me_enckey[1].mv_size);
if (!ekey)
return ENOMEM;
env->me_enckey[0].mv_data = ekey;
env->me_enckey[1].mv_data = ekey + env->me_enckey[0].mv_size;
memcpy(env->me_enckey[1].mv_data, ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_size);
if (*u != env->me_sumsize)
return MDB_BAD_CHECKSUM;
}
#endif
@ -5912,11 +5915,9 @@ mdb_env_envflags(MDB_env *env)
#if MDB_RPAGE_CACHE
if (!env->me_encfunc) {
static mdb_size_t k = (MDB_SIZE_MAX/67*73) | 1;
mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid) * k;
MDB_val keys[2] = { {sizeof(k), &k}, {sizeof(iv), NULL} };
MDB_val key = {sizeof(k), &k};
int rc;
keys[1].mv_data = &iv;
rc = mdb_env_set_encrypt(env, mdb_enctest, keys);
rc = mdb_env_set_encrypt(env, mdb_enctest, &key, 0);
if (rc)
return rc;
}
@ -6300,7 +6301,7 @@ mdb_env_close(MDB_env *env)
mdb_env_close_active(env, 0);
#if MDB_RPAGE_CACHE
free(env->me_enckey[0].mv_data);
free(env->me_enckey.mv_data);
#endif
free(env);
}
@ -6551,7 +6552,35 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
}
#if MDB_RPAGE_CACHE
static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
static int
mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
{
int rc = 0;
if (env->me_encfunc) {
unsigned short muse = id3->muse;
rc = mdb_rpage_decrypt(env, id3, rem, numpgs);
if (!rc && env->me_sumfunc && muse != id3->muse) {
MDB_page *p = (MDB_page *)(id3->menc + rem * env->me_psize);
rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
}
} else {
if (!(id3->muse & (1 << rem))) {
MDB_page *p;
int bit;
/* If this is an overflow page, set all use bits to the end */
if (rem + numpgs > MDB_RPAGE_CHUNK)
bit = 0xffff;
else
bit = 1;
id3->muse |= (bit << rem);
p = (MDB_page *)(id3->mptr + rem * env->me_psize);
rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
}
}
return rc;
}
/** Map a read-only page.
* There are two levels of tracking in use, a per-txn list and a per-env list.
@ -6704,8 +6733,9 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
id3.menc = tl[x].menc;
id3.muse = tl[x].muse;
tl[x].mref++;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc) return rc;
tl[x].muse = id3.muse;
}
goto ok;
@ -6799,8 +6829,10 @@ retry:
el[x].muse = id3.muse;
} else {
id3.mid = pg0;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
el[x].muse = id3.muse;
}
pthread_mutex_unlock(&env->me_rpmutex);
@ -6808,8 +6840,10 @@ retry:
}
}
el[x].mref++;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
el[x].muse = id3.muse;
}
pthread_mutex_unlock(&env->me_rpmutex);
@ -6862,7 +6896,11 @@ fail:
rc = ENOMEM;
goto fail;
}
mdb_rpage_decrypt(env, &id3, rem, numpgs);
}
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
}
mdb_mid3l_insert(el, &id3);
pthread_mutex_unlock(&env->me_rpmutex);
@ -6874,22 +6912,50 @@ found:
ok:
base = (char *)(env->me_encfunc ? id3.menc : id3.mptr);
p = (MDB_page *)(base + rem * env->me_psize);
if (env->me_encfunc)
mdb_rpage_decrypt(env, &id3, rem, numpgs);
rc = MDB_SUCCESS;
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
}
#if MDB_DEBUG /* we don't need this check any more */
if (IS_OVERFLOW(p)) {
mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt);
}
#endif
*ret = p;
return MDB_SUCCESS;
return rc;
}
static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t size)
{
MDB_val in, out, enckeys[3];
int xsize = sizeof(pgno_t) + sizeof(txnid_t);
in.mv_size = size - xsize;
in.mv_data = (char *)dp + xsize;
if (env->me_esumsize) {
in.mv_size -= env->me_esumsize;
enckeys[2].mv_size = env->me_esumsize;
enckeys[2].mv_data = in.mv_data + in.mv_size;
} else {
enckeys[2].mv_size = 0;
enckeys[2].mv_data = 0;
}
out.mv_size = in.mv_size;
out.mv_data = (char *)encp + xsize;
encp->mp_pgno = dp->mp_pgno;
encp->mp_txnid = dp->mp_txnid;
enckeys[0] = env->me_enckey;
enckeys[1].mv_size = xsize;
enckeys[1].mv_data = dp;
return env->me_encfunc(&in, &out, enckeys, 1);
}
static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
{
int rc = 0;
if (!(id3->muse & (1 << rem))) {
MDB_val in, out;
MDB_val in, out, enckeys[3];
int bit;
int xsize = sizeof(pgno_t) + sizeof(txnid_t);
/* If this is an overflow page, set all use bits to the end */
if (rem + numpgs > MDB_RPAGE_CHUNK)
@ -6898,12 +6964,32 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
bit = 1;
id3->muse |= (bit << rem);
in.mv_size = numpgs * env->me_psize;
in.mv_data = (char *)id3->mptr + rem * env->me_psize;
in.mv_size = numpgs * env->me_psize - xsize;
in.mv_data = (char *)id3->mptr + rem * env->me_psize + xsize;
enckeys[0] = env->me_enckey;
enckeys[1].mv_size = xsize;
enckeys[1].mv_data = in.mv_data - xsize;
if (env->me_esumsize) {
in.mv_size -= env->me_esumsize;
enckeys[2].mv_size = env->me_esumsize;
enckeys[2].mv_data = in.mv_data + in.mv_size;
} else {
enckeys[2].mv_size = 0;
enckeys[2].mv_data = 0;
}
out.mv_size = in.mv_size;
out.mv_data = (char *)id3->menc + rem * env->me_psize;
env->me_encfunc(&in, &out, env->me_enckey, 0);
out.mv_data = (char *)id3->menc + rem * env->me_psize + xsize;
if (env->me_encfunc(&in, &out, enckeys, 0))
rc = MDB_CRYPTO_FAIL;
else {
MDB_page *penc, *pclr;
penc = (MDB_page *)enckeys[1].mv_data;
pclr = (MDB_page *)(out.mv_data - xsize);
pclr->mp_pgno = penc->mp_pgno;
pclr->mp_txnid = penc->mp_txnid;
}
}
return rc;
}
/** zero out decrypted pages before freeing them */
@ -6923,6 +7009,38 @@ static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
}
free(id3->menc);
}
static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size)
{
MDB_val src, dst, *key;
src.mv_size = size - env->me_sumsize;
src.mv_data = mp;
dst.mv_size = env->me_sumsize;
dst.mv_data = src.mv_data + src.mv_size;
if (env->me_encfunc)
key = &env->me_enckey;
else
key = NULL;
env->me_sumfunc(&src, &dst, key);
}
static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size)
{
MDB_val src, dst, chk, *key;
char sumbuf[256];
src.mv_size = size - env->me_sumsize;
src.mv_data = mp;
chk.mv_size = env->me_sumsize;
chk.mv_data = src.mv_data + src.mv_size;
dst.mv_size = env->me_sumsize;
dst.mv_data = sumbuf;
if (env->me_encfunc)
key = &env->me_enckey;
else
key = NULL;
env->me_sumfunc(&src, &dst, key);
return memcmp(chk.mv_data, dst.mv_data, env->me_sumsize) ? MDB_BAD_CHECKSUM : 0;
}
#endif
/** Find the address of the page corresponding to a given page number.
@ -8759,6 +8877,10 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
np->mp_flags |= flags;
np->mp_lower = (PAGEHDRSZ-PAGEBASE);
np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
#if MDB_RPAGE_CACHE
np->mp_upper -= mc->mc_txn->mt_env->me_sumsize;
np->mp_upper -= mc->mc_txn->mt_env->me_esumsize;
#endif
if (IS_BRANCH(np))
mc->mc_db->md_branch_pages++;
@ -11234,29 +11356,39 @@ mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
#if MDB_RPAGE_CACHE
int ESECT
mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key)
mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size)
{
char *kdata, *ivdata;
char *kdata;
if (!env || !func || !key)
return EINVAL;
if (env->me_flags & MDB_ENV_ACTIVE)
return EINVAL;
if (! (kdata = malloc(key[0].mv_size + key[1].mv_size)))
if (! (kdata = malloc(key[0].mv_size)))
return ENOMEM;
ivdata = kdata + key[0].mv_size;
memcpy(kdata, key[0].mv_data, key[0].mv_size);
memcpy(ivdata, key[1].mv_data, key[1].mv_size);
free(env->me_enckey[0].mv_data);
env->me_enckey[0].mv_data = kdata;
env->me_enckey[0].mv_size = key[0].mv_size;
env->me_enckey[1].mv_data = ivdata;
env->me_enckey[1].mv_size = key[1].mv_size;
memcpy(kdata, key->mv_data, key->mv_size);
free(env->me_enckey.mv_data);
env->me_enckey.mv_data = kdata;
env->me_enckey.mv_size = key->mv_size;
env->me_encfunc = func;
if (size)
env->me_esumsize = size;
env->me_flags |= MDB_REMAP_CHUNKS | MDB_ENCRYPT;
return MDB_SUCCESS;
}
int ESECT
mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
{
if (!env || !func || !size)
return EINVAL;
if (env->me_flags & MDB_ENV_ACTIVE)
return EINVAL;
env->me_sumfunc = func;
env->me_sumsize = size;
return MDB_SUCCESS;
}
#endif
int ESECT

14
libraries/liblmdb/mtest_enc.c
Unescape View File

@ -21,9 +21,10 @@
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
static void encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
{
chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data);
return 0;
}
int main(int argc,char * argv[])
@ -36,11 +37,10 @@ int main(int argc,char * argv[])
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
MDB_val enckey[2];
MDB_val enckey;
int count;
int *values;
char sval[32] = "";
char eiv[] = {3, 1, 4, 1, 5, 9, 2, 6};
char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
@ -53,15 +53,13 @@ int main(int argc,char * argv[])
values[i] = rand()%1024;
}
enckey[0].mv_data = ekey;
enckey[0].mv_size = sizeof(ekey);
enckey[1].mv_data = eiv;
enckey[1].mv_size = sizeof(eiv);
enckey.mv_data = ekey;
enckey.mv_size = sizeof(ekey);
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_encrypt(env, encfunc, enckey));
E(mdb_env_set_encrypt(env, encfunc, &enckey, 0));
E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));

Write Preview
Loading…
Cancel
Save