2 * keydb_db4.c - Routines to store and fetch keys in a DB3 database.
4 * Jonathan McDowell <noodles@earth.li>
6 * Copyright 2002-2004 Project Purple
21 #include "charfuncs.h"
24 #include "decodekey.h"
25 #include "keystructs.h"
28 #include "onak-conf.h"
33 * dbenv - our database environment.
35 static DB_ENV *dbenv = NULL;
38 * numdb - The number of database files we have.
40 static int numdbs = 16;
43 * dbconn - our connections to the key database files.
45 static DB **dbconns = NULL;
48 * worddb - our connection to the word database.
50 static DB *worddb = NULL;
53 * id32db - our connection to the 32bit ID database.
55 static DB *id32db = NULL;
58 * txn - our current transaction id.
60 static DB_TXN *txn = NULL;
62 DB *keydb(uint64_t keyid)
68 return(dbconns[keytrun % numdbs]);
72 * initdb - Initialize the key database.
74 * This function should be called before any of the other functions in
75 * this file are called in order to allow the DB to be initialized ready
78 void initdb(bool readonly)
86 snprintf(buf, sizeof(buf) - 1, "%s/num_keydb", config.db_dir);
87 numdb = fopen(buf, "r");
89 if (fgets(buf, sizeof(buf), numdb) != NULL) {
93 } else if (!readonly) {
94 logthing(LOGTHING_ERROR, "Couldn't open num_keydb: %s",
96 numdb = fopen(buf, "w");
98 fprintf(numdb, "%d", numdbs);
101 logthing(LOGTHING_ERROR,
102 "Couldn't write num_keydb: %s",
107 dbconns = malloc(sizeof (DB *) * numdbs);
108 if (dbconns == NULL) {
109 logthing(LOGTHING_CRITICAL,
110 "Couldn't allocate memory for dbconns");
114 ret = db_env_create(&dbenv, 0);
116 logthing(LOGTHING_CRITICAL,
117 "db_env_create: %s", db_strerror(ret));
122 * Enable deadlock detection so that we don't block indefinitely on
123 * anything. What we really want is simple 2 state locks, but I'm not
124 * sure how to make the standard DB functions do that yet.
126 ret = dbenv->set_lk_detect(dbenv, DB_LOCK_DEFAULT);
128 logthing(LOGTHING_CRITICAL,
129 "db_env_create: %s", db_strerror(ret));
133 ret = dbenv->open(dbenv, config.db_dir,
134 DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_LOCK |
139 logthing(LOGTHING_CRITICAL,
140 "Error opening db environment: %s (%s)",
148 for (i = 0; i < numdbs; i++) {
149 ret = db_create(&dbconns[i], dbenv, 0);
151 logthing(LOGTHING_CRITICAL,
152 "db_create: %s", db_strerror(ret));
156 snprintf(buf, 1023, "keydb.%d.db", i);
161 ret = dbconns[i]->open(dbconns[i],
169 logthing(LOGTHING_CRITICAL,
170 "Error opening key database: %s (%s)",
177 ret = db_create(&worddb, dbenv, 0);
179 logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret));
182 ret = worddb->set_flags(worddb, DB_DUP);
184 ret = worddb->open(worddb, txn, "worddb", "worddb", DB_BTREE,
188 logthing(LOGTHING_CRITICAL,
189 "Error opening word database: %s (%s)",
195 ret = db_create(&id32db, dbenv, 0);
197 logthing(LOGTHING_CRITICAL, "db_create: %s", db_strerror(ret));
200 ret = id32db->set_flags(id32db, DB_DUP);
202 ret = id32db->open(id32db, txn, "id32db", "id32db", DB_HASH,
206 logthing(LOGTHING_CRITICAL,
207 "Error opening id32 database: %s (%s)",
218 * cleanupdb - De-initialize the key database.
220 * This function should be called upon program exit to allow the DB to
221 * cleanup after itself.
228 dbenv->txn_checkpoint(dbenv, 0, 0, 0);
229 if (id32db != NULL) {
230 id32db->close(id32db, 0);
233 if (worddb != NULL) {
234 worddb->close(worddb, 0);
237 for (i = 0; i < numdbs; i++) {
238 if (dbconns[i] != NULL) {
239 dbconns[i]->close(dbconns[i], 0);
243 dbenv->close(dbenv, 0);
249 * starttrans - Start a transaction.
251 * Start a transaction. Intended to be used if we're about to perform many
252 * operations on the database to help speed it all up, or if we want
253 * something to only succeed if all relevant operations are successful.
255 bool starttrans(void)
259 log_assert(dbenv != NULL);
260 log_assert(txn == NULL);
262 ret = dbenv->txn_begin(dbenv,
263 NULL, /* No parent transaction */
267 logthing(LOGTHING_CRITICAL,
268 "Error starting transaction: %s",
277 * endtrans - End a transaction.
279 * Ends a transaction.
285 log_assert(dbenv != NULL);
286 log_assert(txn != NULL);
288 ret = txn->commit(txn,
291 logthing(LOGTHING_CRITICAL,
292 "Error ending transaction: %s",
302 * fetch_key - Given a keyid fetch the key from storage.
303 * @keyid: The keyid to fetch.
304 * @publickey: A pointer to a structure to return the key in.
305 * @intrans: If we're already in a transaction.
307 * We use the hex representation of the keyid as the filename to fetch the
308 * key from. The key is stored in the file as a binary OpenPGP stream of
309 * packets, so we can just use read_openpgp_stream() to read the packets
310 * in and then parse_keys() to parse the packets into a publickey
313 int fetch_key(uint64_t keyid, struct openpgp_publickey **publickey,
316 struct openpgp_packet_list *packets = NULL;
320 struct buffer_ctx fetchbuf;
322 if (keyid < 0x100000000LL) {
323 keyid = getfullkeyid(keyid);
326 memset(&key, 0, sizeof(key));
327 memset(&data, 0, sizeof(data));
332 key.size = sizeof(keyid);
339 ret = keydb(keyid)->get(keydb(keyid),
346 fetchbuf.buffer = data.data;
348 fetchbuf.size = data.size;
349 read_openpgp_stream(buffer_fetchchar, &fetchbuf,
351 parse_keys(packets, publickey);
352 free_packet_list(packets);
355 } else if (ret != DB_NOTFOUND) {
356 logthing(LOGTHING_ERROR,
357 "Problem retrieving key: %s",
368 int worddb_cmp(const void *d1, const void *d2)
370 return memcmp(d1, d2, 12);
374 * fetch_key_text - Trys to find the keys that contain the supplied text.
375 * @search: The text to search for.
376 * @publickey: A pointer to a structure to return the key in.
378 * This function searches for the supplied text and returns the keys that
381 int fetch_key_text(const char *search, struct openpgp_publickey **publickey)
389 char *searchtext = NULL;
390 struct ll *wordlist = NULL;
391 struct ll *curword = NULL;
392 struct ll *keylist = NULL;
393 struct ll *newkeylist = NULL;
396 searchtext = strdup(search);
397 wordlist = makewordlist(wordlist, searchtext);
401 ret = worddb->cursor(worddb,
406 for (curword = wordlist; curword != NULL; curword = curword->next) {
407 memset(&key, 0, sizeof(key));
408 memset(&data, 0, sizeof(data));
409 key.data = curword->object;
410 key.size = strlen(curword->object);
411 data.flags = DB_DBT_MALLOC;
412 ret = cursor->c_get(cursor,
416 while (ret == 0 && strncmp(key.data, curword->object,
418 ((char *) curword->object)[key.size] == 0) {
420 for (i = 4; i < 12; i++) {
422 keyid += ((unsigned char *)
426 if (keylist == NULL ||
427 llfind(keylist, data.data,
428 worddb_cmp) != NULL) {
429 newkeylist = lladd(newkeylist, data.data);
435 ret = cursor->c_get(cursor,
440 llfree(keylist, free);
441 keylist = newkeylist;
443 if (data.data != NULL) {
448 llfree(wordlist, NULL);
451 for (newkeylist = keylist;
452 newkeylist != NULL && numkeys < config.maxkeys;
453 newkeylist = newkeylist->next) {
456 for (i = 4; i < 12; i++) {
458 keyid += ((unsigned char *)
459 newkeylist->object)[i];
462 numkeys += fetch_key(keyid,
466 llfree(keylist, free);
471 ret = cursor->c_close(cursor);
480 * store_key - Takes a key and stores it.
481 * @publickey: A pointer to the public key to store.
482 * @intrans: If we're already in a transaction.
483 * @update: If true the key exists and should be updated.
485 * Again we just use the hex representation of the keyid as the filename
486 * to store the key to. We flatten the public key to a list of OpenPGP
487 * packets and then use write_openpgp_stream() to write the stream out to
488 * the file. If update is true then we delete the old key first, otherwise
489 * we trust that it doesn't exist.
491 int store_key(struct openpgp_publickey *publickey, bool intrans, bool update)
493 struct openpgp_packet_list *packets = NULL;
494 struct openpgp_packet_list *list_end = NULL;
495 struct openpgp_publickey *next = NULL;
498 struct buffer_ctx storebuf;
502 uint32_t shortkeyid = 0;
503 uint64_t *subkeyids = NULL;
505 char *primary = NULL;
506 unsigned char worddb_data[12];
507 struct ll *wordlist = NULL;
508 struct ll *curword = NULL;
509 bool deadlock = false;
511 keyid = get_keyid(publickey);
518 * Delete the key if we already have it.
520 * TODO: Can we optimize this perhaps? Possibly when other data is
521 * involved as well? I suspect this is easiest and doesn't make a lot
522 * of difference though - the largest chunk of data is the keydata and
523 * it definitely needs updated.
526 deadlock = (delete_key(keyid, true) == -1);
530 * Convert the key to a flat set of binary data.
533 next = publickey->next;
534 publickey->next = NULL;
535 flatten_publickey(publickey, &packets, &list_end);
536 publickey->next = next;
539 storebuf.size = 8192;
540 storebuf.buffer = malloc(8192);
542 write_openpgp_stream(buffer_putchar, &storebuf, packets);
545 * Now we have the key data store it in the DB; the keyid is
548 memset(&key, 0, sizeof(key));
549 memset(&data, 0, sizeof(data));
551 key.size = sizeof(keyid);
552 data.size = storebuf.offset;
553 data.data = storebuf.buffer;
555 ret = keydb(keyid)->put(keydb(keyid),
561 logthing(LOGTHING_ERROR,
562 "Problem storing key: %s",
564 if (ret == DB_LOCK_DEADLOCK) {
569 free(storebuf.buffer);
570 storebuf.buffer = NULL;
574 free_packet_list(packets);
579 * Walk through our uids storing the words into the db with the keyid.
582 uids = keyuids(publickey, &primary);
585 for (i = 0; ret == 0 && uids[i] != NULL; i++) {
586 wordlist = makewordlist(wordlist, uids[i]);
589 for (curword = wordlist; curword != NULL && !deadlock;
590 curword = curword->next) {
591 memset(&key, 0, sizeof(key));
592 memset(&data, 0, sizeof(data));
593 key.data = curword->object;
594 key.size = strlen(key.data);
595 data.data = worddb_data;
596 data.size = sizeof(worddb_data);
599 * Our data is the key creation time followed by the
602 worddb_data[ 0] = publickey->publickey->data[1];
603 worddb_data[ 1] = publickey->publickey->data[2];
604 worddb_data[ 2] = publickey->publickey->data[3];
605 worddb_data[ 3] = publickey->publickey->data[4];
606 worddb_data[ 4] = (keyid >> 56) & 0xFF;
607 worddb_data[ 5] = (keyid >> 48) & 0xFF;
608 worddb_data[ 6] = (keyid >> 40) & 0xFF;
609 worddb_data[ 7] = (keyid >> 32) & 0xFF;
610 worddb_data[ 8] = (keyid >> 24) & 0xFF;
611 worddb_data[ 9] = (keyid >> 16) & 0xFF;
612 worddb_data[10] = (keyid >> 8) & 0xFF;
613 worddb_data[11] = keyid & 0xFF;
614 ret = worddb->put(worddb,
620 logthing(LOGTHING_ERROR,
621 "Problem storing word: %s",
623 if (ret == DB_LOCK_DEADLOCK) {
630 * Free our UID and word lists.
632 llfree(wordlist, NULL);
633 for (i = 0; uids[i] != NULL; i++) {
642 * Write the truncated 32 bit keyid so we can lookup the full id for
646 shortkeyid = keyid & 0xFFFFFFFF;
648 memset(&key, 0, sizeof(key));
649 memset(&data, 0, sizeof(data));
650 key.data = &shortkeyid;
651 key.size = sizeof(shortkeyid);
653 data.size = sizeof(keyid);
655 ret = id32db->put(id32db,
661 logthing(LOGTHING_ERROR,
662 "Problem storing short keyid: %s",
664 if (ret == DB_LOCK_DEADLOCK) {
671 subkeyids = keysubkeys(publickey);
673 while (subkeyids != NULL && subkeyids[i] != 0) {
674 shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
676 memset(&key, 0, sizeof(key));
677 memset(&data, 0, sizeof(data));
678 key.data = &shortkeyid;
679 key.size = sizeof(shortkeyid);
681 data.size = sizeof(keyid);
683 ret = id32db->put(id32db,
689 logthing(LOGTHING_ERROR,
690 "Problem storing short keyid: %s",
692 if (ret == DB_LOCK_DEADLOCK) {
697 if (subkeyids != NULL) {
707 return deadlock ? -1 : 0 ;
711 * delete_key - Given a keyid delete the key from storage.
712 * @keyid: The keyid to delete.
713 * @intrans: If we're already in a transaction.
715 * This function deletes a public key from whatever storage mechanism we
716 * are using. Returns 0 if the key existed.
718 int delete_key(uint64_t keyid, bool intrans)
720 struct openpgp_publickey *publickey = NULL;
723 uint32_t shortkeyid = 0;
724 uint64_t *subkeyids = NULL;
728 char *primary = NULL;
729 unsigned char worddb_data[12];
730 struct ll *wordlist = NULL;
731 struct ll *curword = NULL;
732 bool deadlock = false;
738 fetch_key(keyid, &publickey, true);
741 * Walk through the uids removing the words from the worddb.
743 if (publickey != NULL) {
744 uids = keyuids(publickey, &primary);
747 for (i = 0; ret == 0 && uids[i] != NULL; i++) {
748 wordlist = makewordlist(wordlist, uids[i]);
751 ret = worddb->cursor(worddb,
756 for (curword = wordlist; curword != NULL && !deadlock;
757 curword = curword->next) {
758 memset(&key, 0, sizeof(key));
759 memset(&data, 0, sizeof(data));
760 key.data = curword->object;
761 key.size = strlen(key.data);
762 data.data = worddb_data;
763 data.size = sizeof(worddb_data);
766 * Our data is the key creation time followed by the
769 worddb_data[ 0] = publickey->publickey->data[1];
770 worddb_data[ 1] = publickey->publickey->data[2];
771 worddb_data[ 2] = publickey->publickey->data[3];
772 worddb_data[ 3] = publickey->publickey->data[4];
773 worddb_data[ 4] = (keyid >> 56) & 0xFF;
774 worddb_data[ 5] = (keyid >> 48) & 0xFF;
775 worddb_data[ 6] = (keyid >> 40) & 0xFF;
776 worddb_data[ 7] = (keyid >> 32) & 0xFF;
777 worddb_data[ 8] = (keyid >> 24) & 0xFF;
778 worddb_data[ 9] = (keyid >> 16) & 0xFF;
779 worddb_data[10] = (keyid >> 8) & 0xFF;
780 worddb_data[11] = keyid & 0xFF;
782 ret = cursor->c_get(cursor,
788 ret = cursor->c_del(cursor, 0);
790 logthing(LOGTHING_ERROR,
791 "Problem deleting word: %s",
797 logthing(LOGTHING_ERROR,
798 "Problem deleting word: %s",
800 if (ret == DB_LOCK_DEADLOCK) {
805 ret = cursor->c_close(cursor);
809 * Free our UID and word lists.
811 llfree(wordlist, NULL);
812 for (i = 0; uids[i] != NULL; i++) {
818 free_publickey(publickey);
823 ret = id32db->cursor(id32db,
828 shortkeyid = keyid & 0xFFFFFFFF;
830 memset(&key, 0, sizeof(key));
831 memset(&data, 0, sizeof(data));
832 key.data = &shortkeyid;
833 key.size = sizeof(shortkeyid);
835 data.size = sizeof(keyid);
837 ret = cursor->c_get(cursor,
843 ret = cursor->c_del(cursor, 0);
845 logthing(LOGTHING_ERROR,
846 "Problem deleting short keyid: %s",
852 logthing(LOGTHING_ERROR,
853 "Problem deleting short keyid: %s",
855 if (ret == DB_LOCK_DEADLOCK) {
860 subkeyids = keysubkeys(publickey);
862 while (subkeyids != NULL && subkeyids[i] != 0) {
863 shortkeyid = subkeyids[i++] & 0xFFFFFFFF;
865 memset(&key, 0, sizeof(key));
866 memset(&data, 0, sizeof(data));
867 key.data = &shortkeyid;
868 key.size = sizeof(shortkeyid);
870 data.size = sizeof(keyid);
872 ret = cursor->c_get(cursor,
878 ret = cursor->c_del(cursor, 0);
880 logthing(LOGTHING_ERROR,
881 "Problem deleting short"
888 logthing(LOGTHING_ERROR,
889 "Problem deleting short keyid: %s",
891 if (ret == DB_LOCK_DEADLOCK) {
896 if (subkeyids != NULL) {
901 ret = cursor->c_close(cursor);
907 key.size = sizeof(keyid);
909 keydb(keyid)->del(keydb(keyid),
919 return deadlock ? (-1) : (ret == DB_NOTFOUND);
923 * dumpdb - dump the key database
924 * @filenamebase: The base filename to use for the dump.
926 * Dumps the database into one or more files, which contain pure OpenPGP
927 * that can be reimported into onak or gpg. filenamebase provides a base
928 * file name for the dump; several files may be created, all of which will
929 * begin with this string and then have a unique number and a .pgp
932 int dumpdb(char *filenamebase)
942 for (i = 0; i < numdbs; i++) {
943 ret = dbconns[i]->cursor(dbconns[i],
948 snprintf(filename, 1023, "%s.%d.pgp", filenamebase, i);
949 fd = open(filename, O_CREAT | O_WRONLY | O_TRUNC, 0640);
951 logthing(LOGTHING_ERROR,
952 "Error opening keydump file (%s): %s",
956 memset(&key, 0, sizeof(key));
957 memset(&data, 0, sizeof(data));
958 ret = cursor->c_get(cursor, &key, &data, DB_NEXT);
960 write(fd, data.data, data.size);
961 memset(&key, 0, sizeof(key));
962 memset(&data, 0, sizeof(data));
963 ret = cursor->c_get(cursor, &key, &data,
966 if (ret != DB_NOTFOUND) {
967 logthing(LOGTHING_ERROR,
968 "Problem reading key: %s",
974 ret = cursor->c_close(cursor);
982 * iterate_keys - call a function once for each key in the db.
983 * @iterfunc: The function to call.
984 * @ctx: A context pointer
986 * Calls iterfunc once for each key in the database. ctx is passed
987 * unaltered to iterfunc. This function is intended to aid database dumps
988 * and statistic calculations.
990 * Returns the number of keys we iterated over.
992 int iterate_keys(void (*iterfunc)(void *ctx, struct openpgp_publickey *key),
1000 struct buffer_ctx fetchbuf;
1001 struct openpgp_packet_list *packets = NULL;
1002 struct openpgp_publickey *key = NULL;
1004 for (i = 0; i < numdbs; i++) {
1005 ret = dbconns[i]->cursor(dbconns[i],
1010 memset(&dbkey, 0, sizeof(dbkey));
1011 memset(&data, 0, sizeof(data));
1012 ret = cursor->c_get(cursor, &dbkey, &data, DB_NEXT);
1014 fetchbuf.buffer = data.data;
1015 fetchbuf.offset = 0;
1016 fetchbuf.size = data.size;
1017 read_openpgp_stream(buffer_fetchchar, &fetchbuf,
1019 parse_keys(packets, &key);
1023 free_publickey(key);
1025 free_packet_list(packets);
1028 memset(&dbkey, 0, sizeof(dbkey));
1029 memset(&data, 0, sizeof(data));
1030 ret = cursor->c_get(cursor, &dbkey, &data,
1034 if (ret != DB_NOTFOUND) {
1035 logthing(LOGTHING_ERROR,
1036 "Problem reading key: %s",
1040 ret = cursor->c_close(cursor);
1048 * getfullkeyid - Maps a 32bit key id to a 64bit one.
1049 * @keyid: The 32bit keyid.
1051 * This function maps a 32bit key id to the full 64bit one. It returns the
1052 * full keyid. If the key isn't found a keyid of 0 is returned.
1054 uint64_t getfullkeyid(uint64_t keyid)
1058 uint32_t shortkeyid = 0;
1061 if (keyid < 0x100000000LL) {
1062 ret = id32db->cursor(id32db,
1067 shortkeyid = keyid & 0xFFFFFFFF;
1069 memset(&key, 0, sizeof(key));
1070 memset(&data, 0, sizeof(data));
1071 key.data = &shortkeyid;
1072 key.size = sizeof(shortkeyid);
1073 data.flags = DB_DBT_MALLOC;
1075 ret = cursor->c_get(cursor,
1081 keyid = *(uint64_t *) data.data;
1083 if (data.data != NULL) {
1089 ret = cursor->c_close(cursor);
1097 * Include the basic keydb routines.
1099 #define NEED_GETKEYSIGS 1
1100 #define NEED_KEYID2UID 1
1101 #define NEED_UPDATEKEYS 1