[Pkg-gnupg-commit] [gnupg2] 61/241: gpg: Provide an interface to patch TOFU updates.
Daniel Kahn Gillmor
dkg at fifthhorseman.net
Wed Dec 9 20:31:54 UTC 2015
This is an automated email from the git hooks/post-receive script.
dkg pushed a commit to branch master
in repository gnupg2.
commit 7f65e84ac035e8f7a25639a6b09eb6000115e337
Author: Neal H. Walfield <neal at g10code.com>
Date: Fri Oct 23 17:23:17 2015 +0200
gpg: Provide an interface to patch TOFU updates.
* g10/tofu.c (struct db): Rename begin_transaction to savepoint_batch.
Rename end_transaction to savepoint_batch_commit. Update users.
Remove field rollback. Add fields savepoint_inner and
savepoint_inner_commit. Add field batch_update.
(dump_cache): New function.
(batch_update): New variable.
(begin_transaction). New function.
(end_transaction): New function.
(rollback_transaction): New function.
(tofu_begin_batch_update): New function.
(tofu_end_batch_update): New function.
(closedb): End any pending batch transaction.
(closedbs): Assert that none of the DBs have a started batch
transaction if we not in batch mode.
(record_binding): Use the begin_transaction, end_transaction and
rollback_transaction functions instead of including the SQL inline.
Also start a batch mode transaction if we are using the flat format.
(tofu_register): Use the begin_transaction, end_transaction and
rollback_transaction functions instead of including the SQL inline.
* g10/gpgv.c (tofu_begin_batch_update): New function.
(tofu_end_batch_update): New function.
* g10/test-stubs.c (tofu_begin_batch_update): New function.
(tofu_end_batch_update): New function.
--
Signed-off-by: Neal H. Walfield <neal at g10code.com>
---
g10/gpgv.c | 10 +++
g10/keylist.c | 4 +
g10/test-stubs.c | 10 +++
g10/tofu.c | 262 ++++++++++++++++++++++++++++++++++++++++++++-----------
g10/tofu.h | 6 ++
5 files changed, 239 insertions(+), 53 deletions(-)
diff --git a/g10/gpgv.c b/g10/gpgv.c
index 1d7dc74..ec09706 100644
--- a/g10/gpgv.c
+++ b/g10/gpgv.c
@@ -632,3 +632,13 @@ tofu_policy_str (enum tofu_policy policy)
return "unknown";
}
+
+void
+tofu_begin_batch_update (void)
+{
+}
+
+void
+tofu_end_batch_update (void)
+{
+}
diff --git a/g10/keylist.c b/g10/keylist.c
index 2a766a1..d4e6b74 100644
--- a/g10/keylist.c
+++ b/g10/keylist.c
@@ -132,12 +132,16 @@ public_key_list (ctrl_t ctrl, strlist_t list, int locate_mode)
which is associated with the inode of a deleted file. */
check_trustdb_stale ();
+ tofu_begin_batch_update ();
+
if (locate_mode)
locate_one (ctrl, list);
else if (!list)
list_all (ctrl, 0, opt.with_secret);
else
list_one (ctrl, list, 0, opt.with_secret);
+
+ tofu_end_batch_update ();
}
diff --git a/g10/test-stubs.c b/g10/test-stubs.c
index 4edea69..dfe6edb 100644
--- a/g10/test-stubs.c
+++ b/g10/test-stubs.c
@@ -446,3 +446,13 @@ tofu_policy_str (enum tofu_policy policy)
return "unknown";
}
+
+void
+tofu_begin_batch_update (void)
+{
+}
+
+void
+tofu_end_batch_update (void)
+{
+}
diff --git a/g10/tofu.c b/g10/tofu.c
index 3b8ced0..ad61536 100644
--- a/g10/tofu.c
+++ b/g10/tofu.c
@@ -86,9 +86,11 @@ struct db
struct
{
- sqlite3_stmt *begin_transaction;
- sqlite3_stmt *end_transaction;
- sqlite3_stmt *rollback;
+ sqlite3_stmt *savepoint_batch;
+ sqlite3_stmt *savepoint_batch_commit;
+
+ sqlite3_stmt *savepoint_inner;
+ sqlite3_stmt *savepoint_inner_commit;
sqlite3_stmt *record_binding_get_old_policy;
sqlite3_stmt *record_binding_update;
@@ -101,17 +103,35 @@ struct db
sqlite3_stmt *register_insert;
} s;
-
#if DEBUG_TOFU_CACHE
int hits;
#endif
+ int batch_update;
+
/* If TYPE is DB_COMBINED, this is "". Otherwise, it is either the
fingerprint (type == DB_KEY) or the normalized email address
(type == DB_EMAIL). */
char name[1];
};
+static struct db *db_cache;
+static int db_cache_count;
+#define DB_CACHE_ENTRIES 16
+
+static void tofu_cache_dump (struct db *db) GPGRT_ATTR_USED;
+
+static void
+tofu_cache_dump (struct db *db)
+{
+ log_info ("Connection %p:\n", db);
+ for (; db; db = db->next)
+ log_info (" %s: %sbatch mode\n", db->name, db->batch_update ? "" : "NOT ");
+ log_info ("Cache:\n");
+ for (db = db_cache; db; db = db->next)
+ log_info (" %s: %sbatch mode\n", db->name, db->batch_update ? "" : "NOT ");
+}
+
#define STRINGIFY(s) STRINGIFY2(s)
#define STRINGIFY2(s) #s
@@ -400,7 +420,160 @@ sqlite3_stepx (sqlite3 *db,
return rc;
}
+
+static int batch_update;
+
+/* Start a transaction on DB. */
+static gpg_error_t
+begin_transaction (struct db *db, int only_batch)
+{
+ int rc;
+ char *err = NULL;
+
+ /* XXX: In split mode, this can end in deadlock.
+
+ Consider: we have two gpg processes running simultaneously and
+ they each want to lock DB A and B, but in different orders. This
+ will be automatically resolved by causing one of them to return
+ EBUSY and aborting.
+
+ A more intelligent approach would be to commit and retake the
+ batch transaction. This requires a list of all DBs that are
+ currently in batch mode. */
+
+ if (batch_update && ! db->batch_update)
+ {
+ rc = sqlite3_stepx (db->db, &db->s.savepoint_batch,
+ NULL, NULL, &err,
+ "savepoint batch;", SQLITE_ARG_END);
+ if (rc)
+ {
+ log_error
+ (_("error beginning %s transaction on TOFU database '%s': %s\n"),
+ "batch", *db->name ? db->name : "combined", err);
+ sqlite3_free (err);
+ return gpg_error (GPG_ERR_GENERAL);
+ }
+
+ db->batch_update = 1;
+ }
+
+ if (only_batch)
+ return 0;
+
+ rc = sqlite3_stepx (db->db, &db->s.savepoint_inner,
+ NULL, NULL, &err,
+ "savepoint inner;", SQLITE_ARG_END);
+ if (rc)
+ {
+ log_error
+ (_("error beginning %s transaction on TOFU database '%s': %s\n"),
+ "inner", *db->name ? db->name : "combined", err);
+ sqlite3_free (err);
+ return gpg_error (GPG_ERR_GENERAL);
+ }
+
+ return 0;
+}
+
+/* Commit a transaction. If ONLY_BATCH is 1, then this only ends the
+ batch transaction if we have left batch mode. If ONLY_BATCH is 2,
+ this ends any open batch transaction even if we are still in batch
+ mode. */
+static gpg_error_t
+end_transaction (struct db *db, int only_batch)
+{
+ int rc;
+ char *err = NULL;
+
+ if ((! batch_update || only_batch == 2) && db->batch_update)
+ /* The batch transaction is still in open, but we left batch
+ mode. */
+ {
+ db->batch_update = 0;
+
+ rc = sqlite3_stepx (db->db, &db->s.savepoint_batch_commit,
+ NULL, NULL, &err,
+ "release batch;", SQLITE_ARG_END);
+ if (rc)
+ {
+ log_error
+ (_("error committing %s transaction on TOFU database '%s': %s\n"),
+ "batch", *db->name ? db->name : "combined", err);
+ sqlite3_free (err);
+ return gpg_error (GPG_ERR_GENERAL);
+ }
+
+ /* Releasing an outer transaction releases an open inner
+ transactions. We're done. */
+ return 0;
+ }
+
+ if (only_batch)
+ return 0;
+
+ rc = sqlite3_stepx (db->db, &db->s.savepoint_inner_commit,
+ NULL, NULL, &err,
+ "release inner;", SQLITE_ARG_END);
+ if (rc)
+ {
+ log_error
+ (_("error committing %s transaction on TOFU database '%s': %s\n"),
+ "inner", *db->name ? db->name : "combined", err);
+ sqlite3_free (err);
+ return gpg_error (GPG_ERR_GENERAL);
+ }
+
+ return 0;
+}
+static gpg_error_t
+rollback_transaction (struct db *db)
+{
+ int rc;
+ char *err = NULL;
+
+ if (db->batch_update)
+ /* Just undo the most recent update; don't revert any progress
+ made by the batch transaction. */
+ rc = sqlite3_exec (db->db, "rollback to inner;", NULL, NULL, &err);
+ else
+ /* Rollback the whole she-bang. */
+ rc = sqlite3_exec (db->db, "rollback;", NULL, NULL, &err);
+
+ if (rc)
+ {
+ log_error
+ (_("error rolling back inner transaction on TOFU database '%s': %s\n"),
+ *db->name ? db->name : "combined", err);
+ sqlite3_free (err);
+ return gpg_error (GPG_ERR_GENERAL);
+ }
+
+ return 0;
+}
+
+void
+tofu_begin_batch_update (void)
+{
+ batch_update ++;
+}
+
+void
+tofu_end_batch_update (void)
+{
+ assert (batch_update > 0);
+ batch_update --;
+
+ if (batch_update == 0)
+ {
+ struct db *db;
+
+ for (db = db_cache; db; db = db->next)
+ end_transaction (db, 1);
+ }
+}
+
/* Collect results of a select count (*) ...; style query. Aborts if
the argument is not a valid integer (or real of the form X.0). */
static int
@@ -652,7 +825,7 @@ initdb (sqlite3 *db, enum db_type type)
}
else
{
- rc = sqlite3_exec (db, "commit transaction;", NULL, NULL, &err);
+ rc = sqlite3_exec (db, "end transaction;", NULL, NULL, &err);
if (rc)
{
log_error (_("error committing transaction on TOFU DB: %s\n"),
@@ -736,10 +909,6 @@ link_db (struct db **head, struct db *db)
*head = db;
}
-static struct db *db_cache;
-static int db_cache_count;
-#define DB_CACHE_ENTRIES 16
-
/* Return a database handle. <type, name> describes the required
database. If there is a cached handle in DBS, that handle is
returned. Otherwise, the database is opened and cached in DBS.
@@ -889,7 +1058,10 @@ closedb (struct db *db)
assert (db->name[0]);
}
- for (statements = &db->s.begin_transaction;
+ if (db->batch_update)
+ end_transaction (db, 2);
+
+ for (statements = (void *) &db->s;
(void *) statements < (void *) &(&db->s)[1];
statements ++)
sqlite3_finalize (*statements);
@@ -983,7 +1155,14 @@ closedbs (struct dbs *dbs)
/* Find the last DB. */
for (db = dbs->db, count = 1; db->next; db = db->next, count ++)
- ;
+ {
+ /* When we leave batch mode we leave batch mode on any
+ cached connections. */
+ if (! batch_update)
+ assert (! db->batch_update);
+ }
+ if (! batch_update)
+ assert (! db->batch_update);
/* Join the two lists. */
db->next = db_cache;
@@ -1084,29 +1263,22 @@ record_binding (struct dbs *dbs, const char *fingerprint, const char *email,
if (! db_key)
return gpg_error (GPG_ERR_GENERAL);
- rc = sqlite3_stepx (db_email->db, &db_email->s.begin_transaction,
- NULL, NULL, &err,
- "begin transaction;", SQLITE_ARG_END);
+ rc = begin_transaction (db_email, 0);
if (rc)
- {
- log_error (_("error beginning transaction on TOFU %s database: %s\n"),
- "email", err);
- sqlite3_free (err);
- return gpg_error (GPG_ERR_GENERAL);
- }
+ return gpg_error (GPG_ERR_GENERAL);
- rc = sqlite3_stepx (db_key->db, &db_key->s.begin_transaction,
- NULL, NULL, &err,
- "begin transaction;", SQLITE_ARG_END);
+ rc = begin_transaction (db_key, 0);
if (rc)
- {
- log_error (_("error beginning transaction on TOFU %s database: %s\n"),
- "key", err);
- sqlite3_free (err);
- goto out_revert_one;
- }
+ goto out_revert_one;
+ }
+ else
+ {
+ rc = begin_transaction (db_email, 1);
+ if (rc)
+ return gpg_error (GPG_ERR_GENERAL);
}
+
if (show_old)
/* Get the old policy. Since this is just for informational
purposes, there is no need to start a transaction or to die if
@@ -1206,13 +1378,9 @@ record_binding (struct dbs *dbs, const char *fingerprint, const char *email,
int rc2;
if (rc)
- rc2 = sqlite3_stepx (db_key->db, &db_key->s.rollback,
- NULL, NULL, &err,
- "rollback;", SQLITE_ARG_END);
+ rc2 = rollback_transaction (db_key);
else
- rc2 = sqlite3_stepx (db_key->db, &db_key->s.end_transaction,
- NULL, NULL, &err,
- "end transaction;", SQLITE_ARG_END);
+ rc2 = end_transaction (db_key, 0);
if (rc2)
{
log_error (_("error ending transaction on TOFU database: %s\n"),
@@ -1222,13 +1390,9 @@ record_binding (struct dbs *dbs, const char *fingerprint, const char *email,
out_revert_one:
if (rc)
- rc2 = sqlite3_stepx (db_email->db, &db_email->s.rollback,
- NULL, NULL, &err,
- "rollback;", SQLITE_ARG_END);
+ rc2 = rollback_transaction (db_email);
else
- rc2 = sqlite3_stepx (db_email->db, &db_email->s.end_transaction,
- NULL, NULL, &err,
- "end transaction;", SQLITE_ARG_END);
+ rc2 = end_transaction (db_email, 0);
if (rc2)
{
log_error (_("error ending transaction on TOFU database: %s\n"),
@@ -2524,15 +2688,9 @@ tofu_register (const byte *fingerprint_bin, const char *user_id,
/* We do a query and then an insert. Make sure they are atomic
by wrapping them in a transaction. */
- rc = sqlite3_stepx (db->db, &db->s.begin_transaction,
- NULL, NULL, &err, "begin transaction;",
- SQLITE_ARG_END);
+ rc = begin_transaction (db, 0);
if (rc)
- {
- log_error (_("error beginning transaction on TOFU database: %s\n"), err);
- sqlite3_free (err);
- goto die;
- }
+ goto die;
/* If we've already seen this signature before, then don't add
it again. */
@@ -2607,11 +2765,9 @@ tofu_register (const byte *fingerprint_bin, const char *user_id,
/* It only matters whether we abort or commit the transaction
(so long as we do something) if we execute the insert. */
if (rc)
- rc = sqlite3_stepx (db->db, &db->s.rollback, NULL, NULL, &err,
- "rollback;", SQLITE_ARG_END);
+ rc = rollback_transaction (db);
else
- rc = sqlite3_stepx (db->db, &db->s.end_transaction, NULL, NULL, &err,
- "end transaction;", SQLITE_ARG_END);
+ rc = end_transaction (db, 0);
if (rc)
{
log_error (_("error ending transaction on TOFU database: %s\n"), err);
diff --git a/g10/tofu.h b/g10/tofu.h
index adf87ab..2d23e86 100644
--- a/g10/tofu.h
+++ b/g10/tofu.h
@@ -106,4 +106,10 @@ gpg_error_t tofu_set_policy_by_keyid (u32 *keyid, enum tofu_policy policy);
gpg_error_t tofu_get_policy (PKT_public_key *pk, PKT_user_id *user_id,
enum tofu_policy *policy);
+/* When doing a lot of DB activities (in particular, when listing
+ keys), this causes the DB to enter batch mode, which can
+ significantly speed up operations. */
+void tofu_begin_batch_update (void);
+void tofu_end_batch_update (void);
+
#endif /*G10_TOFU_H*/
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/pkg-gnupg/gnupg2.git
More information about the Pkg-gnupg-commit
mailing list