ITS#9364 rework crypto API

And add support for per-page checksums. Reserve space for checksum
at tail of page. Pass pgno+txnid as IV input for encryption.
mdb.master3
Howard Chu 4 years ago
parent d1814f7e5d
commit 8dc526c54f
  1. 43
      libraries/liblmdb/lmdb.h
  2. 250
      libraries/liblmdb/mdb.c
  3. 14
      libraries/liblmdb/mtest_enc.c

@ -317,14 +317,26 @@ typedef void (MDB_rel_func)(MDB_val *item, void *oldptr, void *newptr, void *rel
*
* Encrypt or decrypt the data in src and store the result in dst using the
* provided key. The result must be the same number of bytes as the input.
* The input size will always be a multiple of the page size.
* @param[in] src The input data to be transformed.
* @param[out] dst Storage for the result.
* @param[in] key An array of two values: key[0] is the encryption key,
* and key[1] is the initialization vector.
* @param[in] key An array of three values: key[0] is the encryption key,
* key[1] is the initialization vector, and key[2] is the authentication
* data, if any.
* @param[in] encdec 1 to encrypt, 0 to decrypt.
* @return A non-zero error value on failure and 0 on success.
*/
typedef void (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
typedef int (MDB_enc_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec);
/** @brief A callback function used to checksum pages in the env.
*
* Compute the checksum of the data in src and store the result in dst,
* An optional key may be used with keyed hash algorithms.
* @param[in] src The input data to be transformed.
* @param[out] dst Storage for the result.
* @param[in] key An encryption key, if encryption was configured. This
* parameter will be NULL if there is no key.
*/
typedef void (MDB_sum_func)(const MDB_val *src, MDB_val *dst, const MDB_val *key);
#endif
/** @defgroup mdb_env Environment Flags
@ -506,8 +518,12 @@ typedef enum MDB_cursor_op {
#define MDB_BAD_DBI (-30780)
/** Unexpected problem - txn should abort */
#define MDB_PROBLEM (-30779)
/** Page checksum incorrect */
#define MDB_BAD_CHECKSUM (-30778)
/** Encryption/decryption failed */
#define MDB_CRYPTO_FAIL (-30777)
/** The last defined error code */
#define MDB_LAST_ERRCODE MDB_PROBLEM
#define MDB_LAST_ERRCODE MDB_CRYPTO_FAIL
/** @} */
/** @brief Statistics for a database in the environment */
@ -1017,11 +1033,22 @@ int mdb_env_set_assert(MDB_env *env, MDB_assert_func *func);
* It implicitly sets #MDB_REMAP_CHUNKS on the env.
* @param[in] env An environment handle returned by #mdb_env_create().
* @param[in] func An #MDB_enc_func function.
* @param[in] key An array of two values: key[0] is the encryption key,
* and key[1] is the initialization vector.
* @param[in] key The encryption key.
* @param[in] size The size of authentication data in bytes, if any.
* Set this to zero for unauthenticated encryption mechanisms.
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size);
/** @brief Set checksums on an environment.
*
* This must be called before #mdb_env_open().
* @param[in] env An environment handle returned by #mdb_env_create().
* @param[in] func An #MDB_sum_func function.
* @param[in] size The size of computed checksum values, in bytes.
* @return A non-zero error value on failure and 0 on success.
*/
int mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key);
int mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size);
#endif
/** @brief Create a transaction for use with the environment.

@ -1048,7 +1048,7 @@ typedef struct MDB_page {
#define METADATA(p) ((void *)((char *)(p) + PAGEHDRSZ))
/** ITS#7713, change PAGEBASE to handle 65536 byte pages */
#define PAGEBASE ((MDB_DEVEL) ? PAGEHDRSZ : 0)
#define PAGEBASE PAGEHDRSZ
/** Number of nodes on a page */
#define NUMKEYS(p) (((p)->mp_lower - (PAGEHDRSZ-PAGEBASE)) >> 1)
@ -1649,11 +1649,15 @@ struct MDB_env {
#if MDB_RPAGE_CACHE
MDB_ID3L me_rpages; /**< like #mt_rpages, but global to env */
pthread_mutex_t me_rpmutex; /**< control access to #me_rpages */
MDB_sum_func *me_sumfunc; /**< checksum env data */
unsigned short me_sumsize; /**< size of per-page checksums */
#define MDB_ERPAGE_SIZE 16384
#define MDB_ERPAGE_MAX (MDB_ERPAGE_SIZE-1)
unsigned short me_esumsize; /**< size of per-page authentication data */
unsigned int me_rpcheck;
MDB_enc_func *me_encfunc; /**< encrypt env data */
MDB_val me_enckey[2]; /**< key and IV for env encryption */
MDB_val me_enckey; /**< key for env encryption */
#endif
void *me_userctx; /**< User-settable context */
MDB_assert_func *me_assert_func; /**< Callback for assertion failures */
@ -3911,6 +3915,13 @@ mdb_freelist_save(MDB_txn *txn)
return rc;
}
#if MDB_RPAGE_CACHE
static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
static int mdb_page_encrypt(MDB_env *env, MDB_page *in, MDB_page *out, size_t size);
static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size);
static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size);
#endif
/** Flush (some) dirty pages to the map, after clearing their dirty flag.
* @param[in] txn the transaction that's being committed
* @param[in] keep number of initial pages in dirty_list to keep dirty.
@ -3927,9 +3938,6 @@ mdb_page_flush(MDB_txn *txn, int keep)
MDB_OFF_T pos = 0;
pgno_t pgno = 0;
MDB_page *dp = NULL;
#if MDB_RPAGE_CACHE
MDB_page *encp;
#endif
#ifdef _WIN32
OVERLAPPED *ov, *this_ov;
MDB_page *wdp;
@ -4090,16 +4098,17 @@ retry_seek:
wsize = 0;
}
#if MDB_RPAGE_CACHE
if (env->me_sumfunc) {
mdb_page_set_checksum(env, dp, size);
}
if (env->me_encfunc) {
MDB_val in, out;
encp = mdb_page_malloc(txn, nump, 0);
MDB_page *encp = mdb_page_malloc(txn, nump, 0);
if (!encp)
return ENOMEM;
in.mv_size = size;
in.mv_data = dp;
out.mv_size = size;
out.mv_data = encp;
env->me_encfunc(&in, &out, env->me_enckey, 1);
if (mdb_page_encrypt(env, dp, encp, size)) {
mdb_dpage_free_n(env, encp, nump);
return MDB_CRYPTO_FAIL;
}
mdb_dpage_free_n(env, dp, nump);
dp = encp;
dl[i].mptr = dp;
@ -4580,12 +4589,11 @@ mdb_env_init_meta(MDB_env *env, MDB_meta *meta)
*(MDB_meta *)METADATA(q) = *meta;
#if MDB_RPAGE_CACHE
if ((env->me_flags & MDB_ENCRYPT) && env->me_enckey[1].mv_size) {
/* save the IV in tail of page 0 */
if (env->me_sumsize) {
/* save the checksum size in tail of page 0 */
char *ptr = (char *)q;
unsigned short *u = (unsigned short *)(ptr-2);
*u = env->me_enckey[1].mv_size;
memcpy(ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_data, env->me_enckey[1].mv_size);
*u = env->me_sumsize;
}
#endif
DO_PWRITE(rc, env->me_fd, p, psize * NUM_METAS, len, 0);
@ -5331,17 +5339,12 @@ mdb_env_open2(MDB_env *env, int prev)
return MDB_INCOMPATIBLE;
#if MDB_RPAGE_CACHE
if (!newenv && env->me_flags & MDB_ENCRYPT) {
/* for encrypted env, read IV from tail of page 0 */
char *ptr = env->me_map + env->me_psize, *ekey;
if (!newenv && env->me_sumfunc) {
/* for checksums, check sum size from tail of page 0 */
char *ptr = env->me_map + env->me_psize;
unsigned short *u = (unsigned short *)(ptr - 2);
env->me_enckey[1].mv_size = *u;
ekey = realloc(env->me_enckey[0].mv_data, env->me_enckey[0].mv_size + env->me_enckey[1].mv_size);
if (!ekey)
return ENOMEM;
env->me_enckey[0].mv_data = ekey;
env->me_enckey[1].mv_data = ekey + env->me_enckey[0].mv_size;
memcpy(env->me_enckey[1].mv_data, ptr - 2 - env->me_enckey[1].mv_size, env->me_enckey[1].mv_size);
if (*u != env->me_sumsize)
return MDB_BAD_CHECKSUM;
}
#endif
@ -5912,11 +5915,9 @@ mdb_env_envflags(MDB_env *env)
#if MDB_RPAGE_CACHE
if (!env->me_encfunc) {
static mdb_size_t k = (MDB_SIZE_MAX/67*73) | 1;
mdb_size_t iv = ((mdb_size_t)env ^ env->me_pid) * k;
MDB_val keys[2] = { {sizeof(k), &k}, {sizeof(iv), NULL} };
MDB_val key = {sizeof(k), &k};
int rc;
keys[1].mv_data = &iv;
rc = mdb_env_set_encrypt(env, mdb_enctest, keys);
rc = mdb_env_set_encrypt(env, mdb_enctest, &key, 0);
if (rc)
return rc;
}
@ -6300,7 +6301,7 @@ mdb_env_close(MDB_env *env)
mdb_env_close_active(env, 0);
#if MDB_RPAGE_CACHE
free(env->me_enckey[0].mv_data);
free(env->me_enckey.mv_data);
#endif
free(env);
}
@ -6551,7 +6552,35 @@ mdb_cursor_push(MDB_cursor *mc, MDB_page *mp)
}
#if MDB_RPAGE_CACHE
static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs);
static int
mdb_rpage_encsum(MDB_env *env, MDB_ID3 *id3, unsigned rem, int numpgs)
{
int rc = 0;
if (env->me_encfunc) {
unsigned short muse = id3->muse;
rc = mdb_rpage_decrypt(env, id3, rem, numpgs);
if (!rc && env->me_sumfunc && muse != id3->muse) {
MDB_page *p = (MDB_page *)(id3->menc + rem * env->me_psize);
rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
}
} else {
if (!(id3->muse & (1 << rem))) {
MDB_page *p;
int bit;
/* If this is an overflow page, set all use bits to the end */
if (rem + numpgs > MDB_RPAGE_CHUNK)
bit = 0xffff;
else
bit = 1;
id3->muse |= (bit << rem);
p = (MDB_page *)(id3->mptr + rem * env->me_psize);
rc = mdb_page_chk_checksum(env, p, numpgs * env->me_psize);
}
}
return rc;
}
/** Map a read-only page.
* There are two levels of tracking in use, a per-txn list and a per-env list.
@ -6704,8 +6733,9 @@ mdb_rpage_get(MDB_txn *txn, pgno_t pg0, int numpgs, MDB_page **ret)
id3.menc = tl[x].menc;
id3.muse = tl[x].muse;
tl[x].mref++;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc) return rc;
tl[x].muse = id3.muse;
}
goto ok;
@ -6799,8 +6829,10 @@ retry:
el[x].muse = id3.muse;
} else {
id3.mid = pg0;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
el[x].muse = id3.muse;
}
pthread_mutex_unlock(&env->me_rpmutex);
@ -6808,8 +6840,10 @@ retry:
}
}
el[x].mref++;
if (env->me_encfunc) {
mdb_rpage_decrypt(env, &id3, rem, numpgs);
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
el[x].muse = id3.muse;
}
pthread_mutex_unlock(&env->me_rpmutex);
@ -6862,7 +6896,11 @@ fail:
rc = ENOMEM;
goto fail;
}
mdb_rpage_decrypt(env, &id3, rem, numpgs);
}
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
if (rc)
goto fail;
}
mdb_mid3l_insert(el, &id3);
pthread_mutex_unlock(&env->me_rpmutex);
@ -6874,22 +6912,50 @@ found:
ok:
base = (char *)(env->me_encfunc ? id3.menc : id3.mptr);
p = (MDB_page *)(base + rem * env->me_psize);
if (env->me_encfunc)
mdb_rpage_decrypt(env, &id3, rem, numpgs);
rc = MDB_SUCCESS;
if (env->me_encfunc || env->me_sumfunc) {
rc = mdb_rpage_encsum(env, &id3, rem, numpgs);
}
#if MDB_DEBUG /* we don't need this check any more */
if (IS_OVERFLOW(p)) {
mdb_tassert(txn, p->mp_pages + rem <= id3.mcnt);
}
#endif
*ret = p;
return MDB_SUCCESS;
return rc;
}
static int mdb_page_encrypt(MDB_env *env, MDB_page *dp, MDB_page *encp, size_t size)
{
MDB_val in, out, enckeys[3];
int xsize = sizeof(pgno_t) + sizeof(txnid_t);
in.mv_size = size - xsize;
in.mv_data = (char *)dp + xsize;
if (env->me_esumsize) {
in.mv_size -= env->me_esumsize;
enckeys[2].mv_size = env->me_esumsize;
enckeys[2].mv_data = in.mv_data + in.mv_size;
} else {
enckeys[2].mv_size = 0;
enckeys[2].mv_data = 0;
}
out.mv_size = in.mv_size;
out.mv_data = (char *)encp + xsize;
encp->mp_pgno = dp->mp_pgno;
encp->mp_txnid = dp->mp_txnid;
enckeys[0] = env->me_enckey;
enckeys[1].mv_size = xsize;
enckeys[1].mv_data = dp;
return env->me_encfunc(&in, &out, enckeys, 1);
}
static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
static int mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
{
int rc = 0;
if (!(id3->muse & (1 << rem))) {
MDB_val in, out;
MDB_val in, out, enckeys[3];
int bit;
int xsize = sizeof(pgno_t) + sizeof(txnid_t);
/* If this is an overflow page, set all use bits to the end */
if (rem + numpgs > MDB_RPAGE_CHUNK)
@ -6898,12 +6964,32 @@ static void mdb_rpage_decrypt(MDB_env *env, MDB_ID3 *id3, int rem, int numpgs)
bit = 1;
id3->muse |= (bit << rem);
in.mv_size = numpgs * env->me_psize;
in.mv_data = (char *)id3->mptr + rem * env->me_psize;
in.mv_size = numpgs * env->me_psize - xsize;
in.mv_data = (char *)id3->mptr + rem * env->me_psize + xsize;
enckeys[0] = env->me_enckey;
enckeys[1].mv_size = xsize;
enckeys[1].mv_data = in.mv_data - xsize;
if (env->me_esumsize) {
in.mv_size -= env->me_esumsize;
enckeys[2].mv_size = env->me_esumsize;
enckeys[2].mv_data = in.mv_data + in.mv_size;
} else {
enckeys[2].mv_size = 0;
enckeys[2].mv_data = 0;
}
out.mv_size = in.mv_size;
out.mv_data = (char *)id3->menc + rem * env->me_psize;
env->me_encfunc(&in, &out, env->me_enckey, 0);
out.mv_data = (char *)id3->menc + rem * env->me_psize + xsize;
if (env->me_encfunc(&in, &out, enckeys, 0))
rc = MDB_CRYPTO_FAIL;
else {
MDB_page *penc, *pclr;
penc = (MDB_page *)enckeys[1].mv_data;
pclr = (MDB_page *)(out.mv_data - xsize);
pclr->mp_pgno = penc->mp_pgno;
pclr->mp_txnid = penc->mp_txnid;
}
}
return rc;
}
/** zero out decrypted pages before freeing them */
@ -6923,6 +7009,38 @@ static void mdb_rpage_dispose(MDB_env *env, MDB_ID3 *id3)
}
free(id3->menc);
}
static void mdb_page_set_checksum(MDB_env *env, MDB_page *mp, size_t size)
{
MDB_val src, dst, *key;
src.mv_size = size - env->me_sumsize;
src.mv_data = mp;
dst.mv_size = env->me_sumsize;
dst.mv_data = src.mv_data + src.mv_size;
if (env->me_encfunc)
key = &env->me_enckey;
else
key = NULL;
env->me_sumfunc(&src, &dst, key);
}
static int mdb_page_chk_checksum(MDB_env *env, MDB_page *mp, size_t size)
{
MDB_val src, dst, chk, *key;
char sumbuf[256];
src.mv_size = size - env->me_sumsize;
src.mv_data = mp;
chk.mv_size = env->me_sumsize;
chk.mv_data = src.mv_data + src.mv_size;
dst.mv_size = env->me_sumsize;
dst.mv_data = sumbuf;
if (env->me_encfunc)
key = &env->me_enckey;
else
key = NULL;
env->me_sumfunc(&src, &dst, key);
return memcmp(chk.mv_data, dst.mv_data, env->me_sumsize) ? MDB_BAD_CHECKSUM : 0;
}
#endif
/** Find the address of the page corresponding to a given page number.
@ -8759,6 +8877,10 @@ mdb_page_new(MDB_cursor *mc, uint32_t flags, int num, MDB_page **mp)
np->mp_flags |= flags;
np->mp_lower = (PAGEHDRSZ-PAGEBASE);
np->mp_upper = mc->mc_txn->mt_env->me_psize - PAGEBASE;
#if MDB_RPAGE_CACHE
np->mp_upper -= mc->mc_txn->mt_env->me_sumsize;
np->mp_upper -= mc->mc_txn->mt_env->me_esumsize;
#endif
if (IS_BRANCH(np))
mc->mc_db->md_branch_pages++;
@ -11234,29 +11356,39 @@ mdb_env_set_assert(MDB_env *env, MDB_assert_func *func)
#if MDB_RPAGE_CACHE
int ESECT
mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key)
mdb_env_set_encrypt(MDB_env *env, MDB_enc_func *func, const MDB_val *key, unsigned int size)
{
char *kdata, *ivdata;
char *kdata;
if (!env || !func || !key)
return EINVAL;
if (env->me_flags & MDB_ENV_ACTIVE)
return EINVAL;
if (! (kdata = malloc(key[0].mv_size + key[1].mv_size)))
if (! (kdata = malloc(key[0].mv_size)))
return ENOMEM;
ivdata = kdata + key[0].mv_size;
memcpy(kdata, key[0].mv_data, key[0].mv_size);
memcpy(ivdata, key[1].mv_data, key[1].mv_size);
free(env->me_enckey[0].mv_data);
env->me_enckey[0].mv_data = kdata;
env->me_enckey[0].mv_size = key[0].mv_size;
env->me_enckey[1].mv_data = ivdata;
env->me_enckey[1].mv_size = key[1].mv_size;
memcpy(kdata, key->mv_data, key->mv_size);
free(env->me_enckey.mv_data);
env->me_enckey.mv_data = kdata;
env->me_enckey.mv_size = key->mv_size;
env->me_encfunc = func;
if (size)
env->me_esumsize = size;
env->me_flags |= MDB_REMAP_CHUNKS | MDB_ENCRYPT;
return MDB_SUCCESS;
}
int ESECT
mdb_env_set_checksum(MDB_env *env, MDB_sum_func *func, unsigned int size)
{
if (!env || !func || !size)
return EINVAL;
if (env->me_flags & MDB_ENV_ACTIVE)
return EINVAL;
env->me_sumfunc = func;
env->me_sumsize = size;
return MDB_SUCCESS;
}
#endif
int ESECT

@ -21,9 +21,10 @@
#define CHECK(test, msg) ((test) ? (void)0 : ((void)fprintf(stderr, \
"%s:%d: %s: %s\n", __FILE__, __LINE__, msg, mdb_strerror(rc)), abort()))
static void encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
static int encfunc(const MDB_val *src, MDB_val *dst, const MDB_val *key, int encdec)
{
chacha8(src->mv_data, src->mv_size, key[0].mv_data, key[1].mv_data, dst->mv_data);
return 0;
}
int main(int argc,char * argv[])
@ -36,11 +37,10 @@ int main(int argc,char * argv[])
MDB_stat mst;
MDB_cursor *cursor, *cur2;
MDB_cursor_op op;
MDB_val enckey[2];
MDB_val enckey;
int count;
int *values;
char sval[32] = "";
char eiv[] = {3, 1, 4, 1, 5, 9, 2, 6};
char ekey[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32};
@ -53,15 +53,13 @@ int main(int argc,char * argv[])
values[i] = rand()%1024;
}
enckey[0].mv_data = ekey;
enckey[0].mv_size = sizeof(ekey);
enckey[1].mv_data = eiv;
enckey[1].mv_size = sizeof(eiv);
enckey.mv_data = ekey;
enckey.mv_size = sizeof(ekey);
E(mdb_env_create(&env));
E(mdb_env_set_maxreaders(env, 1));
E(mdb_env_set_mapsize(env, 10485760));
E(mdb_env_set_encrypt(env, encfunc, enckey));
E(mdb_env_set_encrypt(env, encfunc, &enckey, 0));
E(mdb_env_open(env, "./testdb", 0 /*|MDB_NOSYNC*/, 0664));
E(mdb_txn_begin(env, NULL, 0, &txn));

Loading…
Cancel
Save