CHROMIUM: verity: only requeue if necessary

drivers/md/dm-verity.c

 /*
  * Originally based on dm-crypt.c,
  * Copyright (C) 2003 Christophe Saout <christophe@saout.de>
  * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org>
  * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved.
  * Copyright (C) 2010 The Chromium OS Authors <chromium-os-dev@chromium.org>
  *                    All Rights Reserved.
  *
  * This file is released under the GPL.
  *
@@ skipping 164 matching lines @@
 
         char hash_alg[CRYPTO_MAX_ALG_NAME];
 
         int error_behavior;
 
         struct verity_stats stats;
 };
 
 static struct kmem_cache *_verity_io_pool;
 
-static void kverityd_queue_verify(struct dm_verity_io *io);
-static void kverityd_queue_io(struct dm_verity_io *io, bool delayed);
+static void kverityd_verify(struct work_struct *work);
+static void kverityd_io(struct work_struct *work);
 static void kverityd_io_bht_populate(struct dm_verity_io *io);
 static void kverityd_io_bht_populate_end(struct bio *, int error);
 
 static BLOCKING_NOTIFIER_HEAD(verity_error_notifier);
 
 /*-----------------------------------------------
  * Statistic tracking functions
  *-----------------------------------------------*/
 
 void verity_stats_pending_bio_inc(struct verity_config *vc)
@@ skipping 400 matching lines @@
 {
         struct verity_config *vc = io->target->private;
 
         if (io->error)
                 verity_error(vc, io, io->error);
 
         bio_endio(io->bio, io->error);
         mempool_free(io, vc->io_pool);
 }
 
+/* Check for any missing bht hashes. */
+static bool verity_is_bht_populated(struct dm_verity_io *io)
+{
+        struct verity_config *vc = io->target->private;
+        sector_t count;
+
+        for (count = 0; count < io->count; ++count)
+                if (!dm_bht_is_populated(&vc->bht, io->block + count))
+                        return false;
+
+        return true;
+}
+
 /* verity_dec_pending manages the lifetime of all dm_verity_io structs.
  * Non-bug error handling is centralized through this interface and
  * all passage from workqueue to workqueue.
  */
 static void verity_dec_pending(struct dm_verity_io *io)
 {
         struct verity_config *vc = io->target->private;
         VERITY_BUG_ON(!io, "NULL argument");
 
         DMDEBUG("dec pending %p: %d--", io, atomic_read(&io->pending));
 
         if (!atomic_dec_and_test(&io->pending))
                 goto done;
 
         if (unlikely(io->error))
                 goto io_error;
 
-        if (io->flags & VERITY_IOFLAGS_PENDING) {
+        /* I/Os that were pending may now be ready */
+        if ((io->flags & VERITY_IOFLAGS_PENDING) &&
+            !verity_is_bht_populated(io)) {
                 io->flags &= ~VERITY_IOFLAGS_PENDING;
-                kverityd_queue_io(io, true);
-                DMDEBUG("io %p requeued for io");
+                INIT_DELAYED_WORK(&io->work, kverityd_io);
+                queue_delayed_work(vc->io_queue, &io->work, HZ/10);
+                verity_stats_total_requeues_inc(vc);
+                REQTRACE("Block %llu+ is being requeued for io (io:%p)",
+                         ULL(io->block), io);
         } else {
+                io->flags &= ~VERITY_IOFLAGS_PENDING;
                 verity_stats_io_queue_dec(vc);
                 verity_stats_verify_queue_inc(vc);
-                kverityd_queue_verify(io);
-                DMDEBUG("io %p enqueued for verify");
+                INIT_DELAYED_WORK(&io->work, kverityd_verify);
+                queue_delayed_work(vc->verify_queue, &io->work, 0);
+                REQTRACE("Block %llu+ is being queued for verify (io:%p)",
+                         ULL(io->block), io);
         }
 
 done:
         return;
 
 io_error:
         verity_return_bio_to_caller(io);
 }
 
 /* Walks the data set and computes the hash of the data read from the
@@ skipping 60 matching lines @@
                                                work);
         struct verity_config *vc = io->target->private;
 
         io->error = verity_verify(vc, io->bio);
 
         /* Free up the bio and tag with the return value */
         verity_stats_verify_queue_dec(vc);
         verity_return_bio_to_caller(io);
 }
 
-/* After all I/O is completed successfully for a request, it is queued on the
- * verify workqueue to ensure its integrity prior to returning it to the
- * caller.  There may be multiple workqueue threads - one per logical
- * processor.
- */
-static void kverityd_queue_verify(struct dm_verity_io *io)
-{
-        struct verity_config *vc = io->target->private;
-        REQTRACE("Block %llu+ is being queued for verify (io:%p)",
-                 ULL(io->block), io);
-        INIT_DELAYED_WORK(&io->work, kverityd_verify);
-        /* No delay needed. But if we move over jobs with pending io, then
-         * we could probably delay them here.
-         */
-        queue_delayed_work(vc->verify_queue, &io->work, 0);
-}
-
 /* Asynchronously called upon the completion of dm-bht I/O.  The status
  * of the operation is passed back to dm-bht and the next steps are
  * decided by verity_dec_pending.
  */
 static void kverityd_io_bht_populate_end(struct bio *bio, int error)
 {
         struct dm_bht_entry *entry = (struct dm_bht_entry *) bio->bi_private;
         struct dm_verity_io *io = (struct dm_verity_io *) entry->io_context;
 
         DMDEBUG("kverityd_io_bht_populate_end (io:%p, entry:%p)", io, entry);
@@ skipping 222 matching lines @@
                                                work);
         VERITY_BUG_ON(!io->bio);
 
         /* Issue requests asynchronously. */
         verity_inc_pending(io);
         kverityd_src_io_read(io);
         kverityd_io_bht_populate(io);
         verity_dec_pending(io);
 }
 
-/* All incoming requests are queued on the I/O workqueue at least once to
- * acquire both the data from the real device (vc->dev) and any data from the
- * hash tree device (vc->hash_dev) needed to verify the integrity of the data.
- * There may be multiple I/O workqueues - one per logical processor.
- */
-static void kverityd_queue_io(struct dm_verity_io *io, bool delayed)
-{
-        struct verity_config *vc = io->target->private;
-        unsigned long delay = 0;  /* jiffies */
-        /* Send all requests through one call to dm_bht_populate on the
-         * queue to ensure that all blocks are accounted for before
-         * proceeding on to verification.
-         */
-        INIT_DELAYED_WORK(&io->work, kverityd_io);
-        /* If this context is dependent on work from another context, we just
-         * requeue with a delay.  Later we could bounce this work to the verify
-         * queue and have it wait there. TODO(wad)
-         */
-        if (delayed) {
-                verity_stats_total_requeues_inc(vc);
-                delay = HZ / 10;
-                REQTRACE("block %llu+ is being delayed %lu jiffies (io:%p)",
-                         ULL(io->block), delay, io);
-        }
-        queue_delayed_work(vc->io_queue, &io->work, delay);
-}
-
 /* Paired with verity_dec_pending, the pending value in the io dictate the
  * lifetime of a request and when it is ready to be processed on the
  * workqueues.
  */
 static void verity_inc_pending(struct dm_verity_io *io)
 {
         atomic_inc(&io->pending);
 }
 
 /* Block-level requests start here. */
@@ skipping 42 matching lines @@
                 VERITY_BUG_ON(bio->bi_sector % to_sector(VERITY_BLOCK_SIZE));
                 VERITY_BUG_ON(bio->bi_size % VERITY_BLOCK_SIZE);
 
                 /* Queue up the request to be verified */
                 io = verity_io_alloc(ti, bio, bio->bi_sector - ti->begin);
                 if (!io) {
                         DMERR_LIMIT("Failed to allocate and init IO data");
                         return DM_MAPIO_REQUEUE;
                 }
                 verity_stats_io_queue_inc(vc);
-                kverityd_queue_io(io, false);
+                INIT_DELAYED_WORK(&io->work, kverityd_io);
+                queue_delayed_work(vc->io_queue, &io->work, 0);
         }
 
         return DM_MAPIO_SUBMITTED;
 }
 
 /*
  * Non-block interfaces and device-mapper specific code
  */
 
 /*
@@ skipping 359 matching lines @@
         dm_unregister_target(&verity_target);
         kmem_cache_destroy(_verity_io_pool);
 }
 
 module_init(dm_verity_init);
 module_exit(dm_verity_exit);
 
 MODULE_AUTHOR("The Chromium OS Authors <chromium-os-dev@chromium.org>");
 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking");
 MODULE_LICENSE("GPL");