OLD | NEW |
1 /* | 1 /* |
2 * Originally based on dm-crypt.c, | 2 * Originally based on dm-crypt.c, |
3 * Copyright (C) 2003 Christophe Saout <christophe@saout.de> | 3 * Copyright (C) 2003 Christophe Saout <christophe@saout.de> |
4 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> | 4 * Copyright (C) 2004 Clemens Fruhwirth <clemens@endorphin.org> |
5 * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved. | 5 * Copyright (C) 2006-2008 Red Hat, Inc. All rights reserved. |
6 * Copyright (C) 2010 The Chromium OS Authors <chromium-os-dev@chromium.org> | 6 * Copyright (C) 2010 The Chromium OS Authors <chromium-os-dev@chromium.org> |
7 * All Rights Reserved. | 7 * All Rights Reserved. |
8 * | 8 * |
9 * This file is released under the GPL. | 9 * This file is released under the GPL. |
10 * | 10 * |
11 * Implements a verifying transparent block device. | 11 * Implements a verifying transparent block device. |
12 * See Documentation/device-mapper/dm-verity.txt | 12 * See Documentation/device-mapper/dm-verity.txt |
13 */ | 13 */ |
14 #include <linux/async.h> | 14 #include <linux/async.h> |
15 #include <linux/backing-dev.h> | 15 #include <linux/backing-dev.h> |
16 #include <linux/bio.h> | 16 #include <linux/bio.h> |
17 #include <linux/blkdev.h> | 17 #include <linux/blkdev.h> |
18 #include <linux/completion.h> | 18 #include <linux/completion.h> |
19 #include <linux/crypto.h> | |
20 #include <linux/delay.h> | 19 #include <linux/delay.h> |
21 #include <linux/device.h> | 20 #include <linux/device.h> |
22 #include <linux/err.h> | 21 #include <linux/err.h> |
23 #include <linux/genhd.h> | 22 #include <linux/genhd.h> |
24 #include <linux/init.h> | 23 #include <linux/init.h> |
25 #include <linux/jiffies.h> | 24 #include <linux/jiffies.h> |
26 #include <linux/kernel.h> | 25 #include <linux/kernel.h> |
27 #include <linux/mempool.h> | 26 #include <linux/mempool.h> |
28 #include <linux/module.h> | 27 #include <linux/module.h> |
29 #include <linux/slab.h> | 28 #include <linux/slab.h> |
30 #include <linux/workqueue.h> | 29 #include <linux/workqueue.h> |
31 #include <linux/reboot.h> | 30 #include <linux/reboot.h> |
32 #include <asm/atomic.h> | 31 #include <asm/atomic.h> |
33 #include <linux/scatterlist.h> | |
34 #include <asm/page.h> | 32 #include <asm/page.h> |
35 #include <asm/unaligned.h> | 33 #include <asm/unaligned.h> |
36 #include <crypto/hash.h> | |
37 #include <crypto/sha.h> | |
38 | 34 |
39 /* #define CONFIG_DM_DEBUG 1 */ | 35 /* #define CONFIG_DM_DEBUG 1 */ |
40 #define CONFIG_DM_VERITY_TRACE 1 | 36 #define CONFIG_DM_VERITY_TRACE 1 |
41 #include <linux/device-mapper.h> | 37 #include <linux/device-mapper.h> |
42 #include <linux/dm-bht.h> | 38 #include <linux/dm-bht.h> |
43 | 39 |
44 #include "dm-verity.h" | 40 #include "dm-verity.h" |
45 | 41 |
46 #define DM_MSG_PREFIX "verity" | 42 #define DM_MSG_PREFIX "verity" |
47 | 43 |
(...skipping 123 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
171 * in PAGE_SIZE increments. | 167 * in PAGE_SIZE increments. |
172 */ | 168 */ |
173 struct bio_set *bs; | 169 struct bio_set *bs; |
174 | 170 |
175 /* Single threaded I/O submitter */ | 171 /* Single threaded I/O submitter */ |
176 struct workqueue_struct *io_queue; | 172 struct workqueue_struct *io_queue; |
177 /* Multithreaded verifier queue */ | 173 /* Multithreaded verifier queue */ |
178 struct workqueue_struct *verify_queue; | 174 struct workqueue_struct *verify_queue; |
179 | 175 |
180 char hash_alg[CRYPTO_MAX_ALG_NAME]; | 176 char hash_alg[CRYPTO_MAX_ALG_NAME]; |
181 struct hash_desc *hash; /* one per cpu */ | |
182 | 177 |
183 int error_behavior; | 178 int error_behavior; |
184 | 179 |
185 struct verity_stats stats; | 180 struct verity_stats stats; |
186 }; | 181 }; |
187 | 182 |
188 static struct kmem_cache *_verity_io_pool; | 183 static struct kmem_cache *_verity_io_pool; |
189 | 184 |
190 static void kverityd_queue_verify(struct dm_verity_io *io); | 185 static void kverityd_queue_verify(struct dm_verity_io *io); |
191 static void kverityd_queue_io(struct dm_verity_io *io, bool delayed); | 186 static void kverityd_queue_io(struct dm_verity_io *io, bool delayed); |
(...skipping 143 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
335 __bio_clone(clone, bio); | 330 __bio_clone(clone, bio); |
336 clone->bi_private = io; | 331 clone->bi_private = io; |
337 clone->bi_end_io = kverityd_src_io_read_end; | 332 clone->bi_end_io = kverityd_src_io_read_end; |
338 clone->bi_bdev = vc->dev->bdev; | 333 clone->bi_bdev = vc->dev->bdev; |
339 clone->bi_sector = vc->start + io->sector; | 334 clone->bi_sector = vc->start + io->sector; |
340 clone->bi_destructor = dm_verity_bio_destructor; | 335 clone->bi_destructor = dm_verity_bio_destructor; |
341 | 336 |
342 return clone; | 337 return clone; |
343 } | 338 } |
344 | 339 |
345 static int verity_hash_block(struct verity_config *vc, | |
346 struct page *page, | |
347 u8 *digest) | |
348 { | |
349 struct hash_desc *hash_desc = &vc->hash[smp_processor_id()]; | |
350 struct scatterlist sg; | |
351 | |
352 sg_init_table(&sg, 1); | |
353 sg_set_page(&sg, page, VERITY_BLOCK_SIZE, 0); | |
354 | |
355 if (crypto_hash_init(hash_desc)) { | |
356 DMCRIT("Failed to initialize the crypto hash"); | |
357 return -EFAULT; | |
358 } | |
359 | |
360 if (crypto_hash_digest(hash_desc, &sg, VERITY_BLOCK_SIZE, digest)) { | |
361 DMCRIT("crypto_hash_digest failed"); | |
362 return -EFAULT; | |
363 } | |
364 | |
365 return 0; | |
366 } | |
367 | |
368 /* If the request is not successful, this handler takes action. | 340 /* If the request is not successful, this handler takes action. |
369 * TODO make this call a registered handler. | 341 * TODO make this call a registered handler. |
370 */ | 342 */ |
371 static void verity_error(struct verity_config *vc, struct dm_verity_io *io, | 343 static void verity_error(struct verity_config *vc, struct dm_verity_io *io, |
372 int error) | 344 int error) |
373 { | 345 { |
374 const char *message; | 346 const char *message; |
375 int error_behavior = DM_VERITY_ERROR_BEHAVIOR_PANIC; | 347 int error_behavior = DM_VERITY_ERROR_BEHAVIOR_PANIC; |
376 dev_t devt = 0; | 348 dev_t devt = 0; |
377 sector_t block = -1; | 349 sector_t block = -1; |
(...skipping 289 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
667 verity_return_bio_to_caller(io); | 639 verity_return_bio_to_caller(io); |
668 } | 640 } |
669 | 641 |
670 /* Walks the data set and computes the hash of the data read from the | 642 /* Walks the data set and computes the hash of the data read from the |
671 * untrusted source device. The computed hash is then passed to dm-bht | 643 * untrusted source device. The computed hash is then passed to dm-bht |
672 * for verification. | 644 * for verification. |
673 */ | 645 */ |
674 static int verity_verify(struct verity_config *vc, | 646 static int verity_verify(struct verity_config *vc, |
675 struct bio *bio) | 647 struct bio *bio) |
676 { | 648 { |
677 u8 digest[VERITY_MAX_DIGEST_SIZE]; | |
678 int r; | 649 int r; |
679 unsigned int idx, block; | 650 unsigned int idx, block; |
680 unsigned int digest_size = | |
681 crypto_hash_digestsize(vc->hash[smp_processor_id()].tfm); | |
682 | 651 |
683 VERITY_BUG_ON(bio == NULL); | 652 VERITY_BUG_ON(bio == NULL); |
684 | 653 |
685 block = to_bytes(bio->bi_sector) >> VERITY_BLOCK_SHIFT; | 654 block = to_bytes(bio->bi_sector) >> VERITY_BLOCK_SHIFT; |
686 | 655 |
687 for (idx = bio->bi_idx; idx < bio->bi_vcnt; idx++) { | 656 for (idx = bio->bi_idx; idx < bio->bi_vcnt; idx++) { |
688 struct bio_vec *bv = bio_iovec_idx(bio, idx); | 657 struct bio_vec *bv = bio_iovec_idx(bio, idx); |
689 | 658 |
690 VERITY_BUG_ON(bv->bv_offset % VERITY_BLOCK_SIZE); | 659 VERITY_BUG_ON(bv->bv_offset % VERITY_BLOCK_SIZE); |
691 VERITY_BUG_ON(bv->bv_len % VERITY_BLOCK_SIZE); | 660 VERITY_BUG_ON(bv->bv_len % VERITY_BLOCK_SIZE); |
692 | 661 |
693 DMDEBUG("Updating hash for block %u", block); | 662 DMDEBUG("Updating hash for block %u", block); |
694 | 663 |
695 /* TODO(msb) handle case where multiple blocks fit in a page */ | 664 /* TODO(msb) handle case where multiple blocks fit in a page */ |
696 » » r = verity_hash_block(vc, bv->bv_page, digest); | 665 » » r = dm_bht_verify_block(&vc->bht, block, |
697 » » if (r < 0) | 666 » » » » » page_address(bv->bv_page)); |
698 » » » goto bad_hash; | |
699 | |
700 » » r = dm_bht_verify_block(&vc->bht, block, digest, digest_size); | |
701 /* dm_bht functions aren't expected to return errno friendly | 667 /* dm_bht functions aren't expected to return errno friendly |
702 * values. They are converted here for uniformity. | 668 * values. They are converted here for uniformity. |
703 */ | 669 */ |
704 if (r > 0) { | 670 if (r > 0) { |
705 DMERR("Pending data for block %u seen at verify", | 671 DMERR("Pending data for block %u seen at verify", |
706 block); | 672 block); |
707 r = -EBUSY; | 673 r = -EBUSY; |
708 goto bad_state; | 674 goto bad_state; |
709 } | 675 } |
710 if (r < 0) { | 676 if (r < 0) { |
711 DMERR_LIMIT("Block hash does not match!"); | 677 DMERR_LIMIT("Block hash does not match!"); |
712 r = -EACCES; | 678 r = -EACCES; |
713 goto bad_match; | 679 goto bad_match; |
714 } | 680 } |
715 REQTRACE("Block %u verified", block); | 681 REQTRACE("Block %u verified", block); |
716 | 682 |
717 block++; | 683 block++; |
718 /* After completing a block, allow a reschedule. | 684 /* After completing a block, allow a reschedule. |
719 * TODO(wad) determine if this is truly needed. | 685 * TODO(wad) determine if this is truly needed. |
720 */ | 686 */ |
721 cond_resched(); | 687 cond_resched(); |
722 } | 688 } |
723 | 689 |
724 return 0; | 690 return 0; |
725 | 691 |
726 bad_hash: | |
727 bad_state: | 692 bad_state: |
728 bad_match: | 693 bad_match: |
729 return r; | 694 return r; |
730 } | 695 } |
731 | 696 |
732 /* Services the verify workqueue */ | 697 /* Services the verify workqueue */ |
733 static void kverityd_verify(struct work_struct *work) | 698 static void kverityd_verify(struct work_struct *work) |
734 { | 699 { |
735 struct delayed_work *dwork = container_of(work, struct delayed_work, | 700 struct delayed_work *dwork = container_of(work, struct delayed_work, |
736 work); | 701 work); |
(...skipping 388 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1125 * - Improperly sized underlying devices | 1090 * - Improperly sized underlying devices |
1126 * - Out of memory conditions (make sure this isn't too flaky under high load!) | 1091 * - Out of memory conditions (make sure this isn't too flaky under high load!) |
1127 * - Incorrect superhash | 1092 * - Incorrect superhash |
1128 * - Incorrect block hashes | 1093 * - Incorrect block hashes |
1129 * - Incorrect bundle hashes | 1094 * - Incorrect bundle hashes |
1130 * - Boot-up read speed; sustained read speeds | 1095 * - Boot-up read speed; sustained read speeds |
1131 */ | 1096 */ |
1132 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) | 1097 static int verity_ctr(struct dm_target *ti, unsigned int argc, char **argv) |
1133 { | 1098 { |
1134 struct verity_config *vc; | 1099 struct verity_config *vc; |
1135 int cpu = 0; | |
1136 int ret = 0; | 1100 int ret = 0; |
1137 int depth; | 1101 int depth; |
1138 unsigned long long tmpull = 0; | 1102 unsigned long long tmpull = 0; |
1139 sector_t blocks; | 1103 sector_t blocks; |
1140 struct request_queue *dev_queue; | 1104 struct request_queue *dev_queue; |
1141 struct mapped_device *md; | 1105 struct mapped_device *md; |
1142 struct request_queue *md_queue; | 1106 struct request_queue *md_queue; |
1143 | 1107 |
1144 /* Support expanding after the root hash for optional args */ | 1108 /* Support expanding after the root hash for optional args */ |
1145 if (argc < 6) { | 1109 if (argc < 6) { |
(...skipping 79 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1225 * the hash offset was wrong. | 1189 * the hash offset was wrong. |
1226 */ | 1190 */ |
1227 | 1191 |
1228 | 1192 |
1229 /* arg4: cryptographic digest algorithm */ | 1193 /* arg4: cryptographic digest algorithm */ |
1230 if (snprintf(vc->hash_alg, CRYPTO_MAX_ALG_NAME, "%s", argv[4]) >= | 1194 if (snprintf(vc->hash_alg, CRYPTO_MAX_ALG_NAME, "%s", argv[4]) >= |
1231 CRYPTO_MAX_ALG_NAME) { | 1195 CRYPTO_MAX_ALG_NAME) { |
1232 ti->error = "Hash algorithm name is too long"; | 1196 ti->error = "Hash algorithm name is too long"; |
1233 goto bad_hash; | 1197 goto bad_hash; |
1234 } | 1198 } |
1235 /* Allocate enough crypto contexts to be able to perform verifies | |
1236 * on all available CPUs. | |
1237 */ | |
1238 ALLOCTRACE("hash_desc array"); | |
1239 vc->hash = (struct hash_desc *) | |
1240 kcalloc(nr_cpu_ids, sizeof(struct hash_desc), GFP_KERNEL); | |
1241 if (!vc->hash) { | |
1242 DMERR("failed to allocate crypto hash contexts"); | |
1243 return -ENOMEM; | |
1244 } | |
1245 | |
1246 /* Setup the hash first. Its length determines much of the bht layout */ | |
1247 for (cpu = 0; cpu < nr_cpu_ids; ++cpu) { | |
1248 ALLOCTRACE("hash_tfm (per-cpu)"); | |
1249 vc->hash[cpu].tfm = crypto_alloc_hash(vc->hash_alg, 0, 0); | |
1250 if (IS_ERR(vc->hash[cpu].tfm)) { | |
1251 DMERR("failed to allocate crypto hash '%s'", | |
1252 vc->hash_alg); | |
1253 vc->hash[cpu].tfm = NULL; | |
1254 goto bad_hash_alg; | |
1255 } | |
1256 } | |
1257 | 1199 |
1258 /* arg6: override with optional device-specific error behavior */ | 1200 /* arg6: override with optional device-specific error behavior */ |
1259 if (argc >= 7) { | 1201 if (argc >= 7) { |
1260 vc->error_behavior = verity_parse_error_behavior(argv[6]); | 1202 vc->error_behavior = verity_parse_error_behavior(argv[6]); |
1261 } else { | 1203 } else { |
1262 /* Inherit the current global default. */ | 1204 /* Inherit the current global default. */ |
1263 vc->error_behavior = | 1205 vc->error_behavior = |
1264 verity_parse_error_behavior(error_behavior); | 1206 verity_parse_error_behavior(error_behavior); |
1265 } | 1207 } |
1266 if (vc->error_behavior == -1) { | 1208 if (vc->error_behavior == -1) { |
(...skipping 72 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1339 return 0; | 1281 return 0; |
1340 | 1282 |
1341 bad_verify_queue: | 1283 bad_verify_queue: |
1342 destroy_workqueue(vc->io_queue); | 1284 destroy_workqueue(vc->io_queue); |
1343 bad_io_queue: | 1285 bad_io_queue: |
1344 bioset_free(vc->bs); | 1286 bioset_free(vc->bs); |
1345 bad_bs: | 1287 bad_bs: |
1346 mempool_destroy(vc->io_pool); | 1288 mempool_destroy(vc->io_pool); |
1347 bad_slab_pool: | 1289 bad_slab_pool: |
1348 bad_err_behavior: | 1290 bad_err_behavior: |
1349 for (cpu = 0; cpu < nr_cpu_ids; ++cpu) | |
1350 if (vc->hash[cpu].tfm) | |
1351 crypto_free_hash(vc->hash[cpu].tfm); | |
1352 bad_hash_alg: | |
1353 bad_hash: | 1291 bad_hash: |
1354 kfree(vc->hash); | |
1355 dm_put_device(ti, vc->hash_dev); | 1292 dm_put_device(ti, vc->hash_dev); |
1356 bad_hash_dev: | 1293 bad_hash_dev: |
1357 bad_hash_start: | 1294 bad_hash_start: |
1358 dm_put_device(ti, vc->dev); | 1295 dm_put_device(ti, vc->dev); |
1359 bad_depth: | 1296 bad_depth: |
1360 bad_bht: | 1297 bad_bht: |
1361 bad_root_hexdigest: | 1298 bad_root_hexdigest: |
1362 bad_verity_dev: | 1299 bad_verity_dev: |
1363 kfree(vc); /* hash is not secret so no need to zero */ | 1300 kfree(vc); /* hash is not secret so no need to zero */ |
1364 return -EINVAL; | 1301 return -EINVAL; |
1365 } | 1302 } |
1366 | 1303 |
1367 static void verity_dtr(struct dm_target *ti) | 1304 static void verity_dtr(struct dm_target *ti) |
1368 { | 1305 { |
1369 struct verity_config *vc = (struct verity_config *) ti->private; | 1306 struct verity_config *vc = (struct verity_config *) ti->private; |
1370 int cpu; | |
1371 | 1307 |
1372 DMDEBUG("Destroying io_queue"); | 1308 DMDEBUG("Destroying io_queue"); |
1373 destroy_workqueue(vc->io_queue); | 1309 destroy_workqueue(vc->io_queue); |
1374 DMDEBUG("Destroying verify_queue"); | 1310 DMDEBUG("Destroying verify_queue"); |
1375 destroy_workqueue(vc->verify_queue); | 1311 destroy_workqueue(vc->verify_queue); |
1376 | 1312 |
1377 DMDEBUG("Destroying bs"); | 1313 DMDEBUG("Destroying bs"); |
1378 bioset_free(vc->bs); | 1314 bioset_free(vc->bs); |
1379 DMDEBUG("Destroying io_pool"); | 1315 DMDEBUG("Destroying io_pool"); |
1380 mempool_destroy(vc->io_pool); | 1316 mempool_destroy(vc->io_pool); |
1381 DMDEBUG("Destroying crypto hash"); | |
1382 for (cpu = 0; cpu < nr_cpu_ids; ++cpu) | |
1383 if (vc->hash[cpu].tfm) | |
1384 crypto_free_hash(vc->hash[cpu].tfm); | |
1385 kfree(vc->hash); | |
1386 | 1317 |
1387 DMDEBUG("Destroying block hash tree"); | 1318 DMDEBUG("Destroying block hash tree"); |
1388 dm_bht_destroy(&vc->bht); | 1319 dm_bht_destroy(&vc->bht); |
1389 | 1320 |
1390 DMDEBUG("Putting hash_dev"); | 1321 DMDEBUG("Putting hash_dev"); |
1391 dm_put_device(ti, vc->hash_dev); | 1322 dm_put_device(ti, vc->hash_dev); |
1392 | 1323 |
1393 DMDEBUG("Putting dev"); | 1324 DMDEBUG("Putting dev"); |
1394 dm_put_device(ti, vc->dev); | 1325 dm_put_device(ti, vc->dev); |
1395 DMDEBUG("Destroying config"); | 1326 DMDEBUG("Destroying config"); |
(...skipping 104 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1500 dm_unregister_target(&verity_target); | 1431 dm_unregister_target(&verity_target); |
1501 kmem_cache_destroy(_verity_io_pool); | 1432 kmem_cache_destroy(_verity_io_pool); |
1502 } | 1433 } |
1503 | 1434 |
1504 module_init(dm_verity_init); | 1435 module_init(dm_verity_init); |
1505 module_exit(dm_verity_exit); | 1436 module_exit(dm_verity_exit); |
1506 | 1437 |
1507 MODULE_AUTHOR("The Chromium OS Authors <chromium-os-dev@chromium.org>"); | 1438 MODULE_AUTHOR("The Chromium OS Authors <chromium-os-dev@chromium.org>"); |
1508 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); | 1439 MODULE_DESCRIPTION(DM_NAME " target for transparent disk integrity checking"); |
1509 MODULE_LICENSE("GPL"); | 1440 MODULE_LICENSE("GPL"); |
OLD | NEW |