Add stacked backend

This backend takes advantage of the new configuration file flexibility
to enable the stacking of multiple backends together, with each being
tried in turn until the desired keys are found. All stores go to the
first configured backend, and fetches from subsequent backends are
also stored in the first backend.

diff --git a/configure.ac b/configure.ac
index 5286ccc279cafa49bd0fd14cda3efde276a07217..a9ab03cc11fcfd3fb6f5494b4b16a5d6caa3de70 100644 (file)
--- a/configure.ac
+++ b/configure.ac
@@ -24,7 +24,7 @@ AC_CHECK_HEADER([systemd/sd-daemon.h], [
 ])
 
 dnl We should always have these backends available.
-backends="file fs keyd"
+backends="file fs keyd stacked"
 
 LIBCURL_CHECK_CONFIG(,,[have_libcurl="yes" backends="$backends hkp"],have_libcurl="no")
 

diff --git a/keydb_stacked.c b/keydb_stacked.c
new file mode 100644 (file)
index 0000000..c732720
--- /dev/null
+++ b/keydb_stacked.c
@@ -0,0 +1,395 @@
+/*
+ * keydb_stacked.c - backend that stacks other backends together
+ *
+ * Copyright 2016 Jonathan McDowell <noodles@earth.li>
+ *
+ * This program is free software: you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License as published by the Free
+ * Software Foundation; version 2 of the License.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
+ * more details.
+ *
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "decodekey.h"
+#include "hash.h"
+#include "keydb.h"
+#include "keyid.h"
+#include "keystructs.h"
+#include "log.h"
+#include "mem.h"
+#include "merge.h"
+#include "onak-conf.h"
+#include "openpgp.h"
+#include "parsekey.h"
+#include "sendsync.h"
+
+struct onak_stacked_dbctx {
+       struct ll *backends;
+       bool store_on_fallback;
+};
+
+/*
+ * The following functions only apply to the first backend.
+ */
+
+static bool stacked_starttrans(struct onak_dbctx *dbctx)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       return backend->starttrans(backend);
+}
+
+static void stacked_endtrans(struct onak_dbctx *dbctx)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       backend->starttrans(backend);
+}
+
+static int stacked_store_key(struct onak_dbctx *dbctx,
+               struct openpgp_publickey *publickey, bool intrans,
+               bool update)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       return backend->store_key(backend,
+                       publickey, intrans, update);
+}
+
+static int stacked_delete_key(struct onak_dbctx *dbctx, uint64_t keyid,
+               bool intrans)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       return backend->delete_key(backend,
+                       keyid, intrans);
+}
+
+static int stacked_update_keys(struct onak_dbctx *dbctx,
+               struct openpgp_publickey **keys, bool sendsync)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       return backend->update_keys(backend, keys, sendsync);
+}
+
+static int stacked_iterate_keys(struct onak_dbctx *dbctx,
+               void (*iterfunc)(void *ctx, struct openpgp_publickey *key),
+               void *ctx)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+
+       return backend->iterate_keys(backend, iterfunc, ctx);
+}
+
+static void store_on_fallback(struct onak_stacked_dbctx *privctx,
+               struct openpgp_publickey *publickey, bool intrans)
+{
+       struct onak_dbctx *backend =
+                       (struct onak_dbctx *) privctx->backends->object;
+       struct openpgp_publickey *curkey;
+
+       /*
+        * If we walked the stack at all, store the key in the first
+        * backend if configured to do so. It's not an update as we
+        * know it's not there or we wouldn't have fallen back.
+        */
+       for (curkey = publickey; curkey != NULL; curkey = curkey->next) {
+               backend->store_key(backend, curkey, intrans, false);
+       }
+}
+
+/*
+ * The functions below will walk along the backend stack until they
+ * reach the end or get a successful result.
+ */
+
+static int stacked_fetch_key_id(struct onak_dbctx *dbctx, uint64_t keyid,
+               struct openpgp_publickey **publickey, bool intrans)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       int res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->fetch_key_id(backend, keyid, publickey,
+                               intrans);
+       }
+
+       if (privctx->store_on_fallback && cur != privctx->backends) {
+               store_on_fallback(privctx, *publickey, intrans);
+       }
+
+       return res;
+}
+
+static int stacked_fetch_key_fp(struct onak_dbctx *dbctx,
+               struct openpgp_fingerprint *fingerprint,
+               struct openpgp_publickey **publickey, bool intrans)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       int res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->fetch_key_fp(backend, fingerprint, publickey,
+                               intrans);
+       }
+
+       if (privctx->store_on_fallback && cur != privctx->backends) {
+               store_on_fallback(privctx, *publickey, intrans);
+       }
+
+       return res;
+}
+
+static int stacked_fetch_key_text(struct onak_dbctx *dbctx,
+               const char *search,
+               struct openpgp_publickey **publickey)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       int res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->fetch_key_text(backend, search, publickey);
+       }
+
+       if (privctx->store_on_fallback && cur != privctx->backends) {
+               store_on_fallback(privctx, *publickey, false);
+       }
+
+       return res;
+}
+
+static int stacked_fetch_key_skshash(struct onak_dbctx *dbctx,
+               const struct skshash *hash,
+               struct openpgp_publickey **publickey)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       int res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->fetch_key_skshash(backend, hash, publickey);
+       }
+
+       if (privctx->store_on_fallback && cur != privctx->backends) {
+               store_on_fallback(privctx, *publickey, false);
+       }
+
+       return res;
+}
+
+static struct ll *stacked_getkeysigs(struct onak_dbctx *dbctx,
+               uint64_t keyid, bool *revoked)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       struct ll *res = NULL;
+
+       for (cur = privctx->backends; cur != NULL && res == NULL;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->getkeysigs(backend, keyid, revoked);
+       }
+
+       return res;
+}
+
+static struct ll *stacked_cached_getkeysigs(struct onak_dbctx *dbctx,
+               uint64_t keyid)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       struct ll *res = NULL;
+
+       for (cur = privctx->backends; cur != NULL && res == NULL;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->cached_getkeysigs(backend, keyid);
+       }
+
+       return res;
+}
+
+static char *stacked_keyid2uid(struct onak_dbctx *dbctx,
+                       uint64_t keyid)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       char *res = NULL;
+
+       for (cur = privctx->backends; cur != NULL && res == NULL;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->keyid2uid(backend, keyid);
+       }
+
+       return res;
+}
+
+static uint64_t stacked_getfullkeyid(struct onak_dbctx *dbctx,
+               uint64_t keyid)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       uint64_t res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               res = backend->getfullkeyid(backend, keyid);
+       }
+
+       return res;
+}
+
+static void stacked_cleanupdb(struct onak_dbctx *dbctx)
+{
+       struct onak_stacked_dbctx *privctx =
+                       (struct onak_stacked_dbctx *) dbctx->priv;
+       struct onak_dbctx *backend;
+       struct ll *cur;
+       int res = 0;
+
+       for (cur = privctx->backends; cur != NULL && res == 0;
+                       cur = cur->next) {
+               backend = (struct onak_dbctx *) cur->object;
+               backend->cleanupdb(backend);
+       }
+
+       if (dbctx->priv != NULL) {
+               free(dbctx->priv);
+               dbctx->priv = NULL;
+       }
+
+       if (dbctx != NULL) {
+               free(dbctx);
+       }
+}
+
+struct onak_dbctx *keydb_stacked_init(struct onak_db_config *dbcfg,
+               bool readonly)
+{
+       struct onak_dbctx *dbctx;
+       struct onak_stacked_dbctx *privctx;
+       struct onak_dbctx *backend;
+       struct onak_db_config *backend_cfg;
+       char *backend_name, *saveptr = NULL;
+
+       if (dbcfg == NULL) {
+               logthing(LOGTHING_CRITICAL,
+                       "No backend database configuration supplied.");
+               return NULL;
+       }
+
+       dbctx = malloc(sizeof(struct onak_dbctx));
+
+       if (dbctx == NULL) {
+               return NULL;
+       }
+
+       dbctx->config = dbcfg;
+       dbctx->priv = privctx = malloc(sizeof(struct onak_stacked_dbctx));
+       if (dbctx->priv == NULL) {
+               free(dbctx);
+               return (NULL);
+       }
+
+       /* TODO: Make configurable? */
+       privctx->store_on_fallback = true;
+       privctx->backends = NULL;
+
+       backend_name = strtok_r(dbcfg->location, ":", &saveptr);
+       while (backend_name != NULL) {
+               backend_cfg = find_db_backend_config(config.backends,
+                               backend_name);
+               if (backend_cfg == NULL) {
+                       logthing(LOGTHING_CRITICAL,
+                               "Couldn't find configuration for %s backend",
+                               backend_name);
+                       stacked_cleanupdb(dbctx);
+                       return NULL;
+               }
+               logthing(LOGTHING_INFO, "Loading stacked backend: %s",
+                               backend_cfg->name);
+
+               backend = config.dbinit(backend_cfg, readonly);
+               privctx->backends = lladdend(privctx->backends, backend);
+
+               backend_name = strtok_r(NULL, ":", &saveptr);
+       }
+
+       if (privctx->backends != NULL) {
+               dbctx->cleanupdb = stacked_cleanupdb;
+               dbctx->starttrans = stacked_starttrans;
+               dbctx->endtrans = stacked_endtrans;
+               dbctx->fetch_key_id = stacked_fetch_key_id;
+               dbctx->fetch_key_fp = stacked_fetch_key_fp;
+               dbctx->fetch_key_text = stacked_fetch_key_text;
+               dbctx->fetch_key_skshash = stacked_fetch_key_skshash;
+               dbctx->store_key = stacked_store_key;
+               dbctx->update_keys = stacked_update_keys;
+               dbctx->delete_key = stacked_delete_key;
+               dbctx->getkeysigs = stacked_getkeysigs;
+               dbctx->cached_getkeysigs = stacked_cached_getkeysigs;
+               dbctx->keyid2uid = stacked_keyid2uid;
+               dbctx->getfullkeyid = stacked_getfullkeyid;
+               dbctx->iterate_keys = stacked_iterate_keys;
+       }
+
+       return dbctx;
+}

diff --git a/onak-conf.h b/onak-conf.h
index 7c15cbb02583e7d6e2284e04ea7911979f561a14..4cf9bfc2be2b89e9ba99fef6159d980375afcdb8 100644 (file)
--- a/onak-conf.h
+++ b/onak-conf.h
@@ -128,4 +128,10 @@ void writeconfig(const char *configfile);
  */
 void cleanupconfig(void);
 
+
+/**
+ * @brief Find a specified backend configuration by name.
+ */
+struct onak_db_config *find_db_backend_config(struct ll *backends, char *name);
+
 #endif /* __ONAK_CONF_H_ */

diff --git a/onak.ini.in b/onak.ini.in
index 12a8701ef05c0397696440cc10715ebcee30e583..465b4e5362a592843fe6a0f5592ba00b1578bf03 100644 (file)
--- a/onak.ini.in
+++ b/onak.ini.in
@@ -47,5 +47,26 @@ this_site=pgp-public-keys@the.earth.li
 ; Database backend configurations below here
 
 [backend:defaultdb4]
+; The default DB4 backend. Recommended.
 type=db4
 location=@STATEDIR@/lib/onak
+
+[backend:examplehkp]
+; An example HKP backend; all operations will be done against the
+; provided keyserver, with no local storage.
+type=hkp
+location=hkp://the.earth.li/
+
+[backend:examplestacked]
+; A stacked set of backends. All fetch operations will be tried against
+; the provided list of backends, from left to right, until one succeeds.
+; All store operations are against the first backend.
+; If a fetch does not succeed against the first backend, but against a
+; later one, then the returned keys are also stored in the first backend.
+; This example configuration essentially produces a caching keyserver,
+; with any key fetched from the HKP backend being stored in the DB4
+; backend.
+; Note keys are not expired from the DB4 backend, so without any other
+; update mechanism configured this will result in stale data eventually.
+type=stacked
+location=defaultdb4:examplehkp