OLD | NEW |
1 /* | 1 /* |
2 * Copyright 2011 Google Inc. | 2 * Copyright 2011 Google Inc. |
3 * | 3 * |
4 * Use of this source code is governed by a BSD-style license that can be | 4 * Use of this source code is governed by a BSD-style license that can be |
5 * found in the LICENSE file. | 5 * found in the LICENSE file. |
6 */ | 6 */ |
7 | 7 |
8 /* | 8 /* |
9 * Code for the "gm" (Golden Master) rendering comparison tool. | 9 * Code for the "gm" (Golden Master) rendering comparison tool. |
10 * | 10 * |
11 * If you make changes to this, re-run the self-tests at gm/tests/run.sh | 11 * If you make changes to this, re-run the self-tests at gm/tests/run.sh |
12 * to make sure they still pass... you may need to change the expected | 12 * to make sure they still pass... you may need to change the expected |
13 * results of the self-test. | 13 * results of the self-test. |
14 */ | 14 */ |
15 | 15 |
16 #include "gm.h" | 16 #include "gm.h" |
17 #include "gm_error.h" | 17 #include "gm_error.h" |
18 #include "gm_expectations.h" | 18 #include "gm_expectations.h" |
19 #include "system_preferences.h" | 19 #include "system_preferences.h" |
20 #include "SkBitmap.h" | 20 #include "SkBitmap.h" |
21 #include "SkBitmapChecksummer.h" | 21 #include "SkBitmapHasher.h" |
22 #include "SkColorPriv.h" | 22 #include "SkColorPriv.h" |
23 #include "SkCommandLineFlags.h" | 23 #include "SkCommandLineFlags.h" |
24 #include "SkData.h" | 24 #include "SkData.h" |
25 #include "SkDeferredCanvas.h" | 25 #include "SkDeferredCanvas.h" |
26 #include "SkDevice.h" | 26 #include "SkDevice.h" |
27 #include "SkDrawFilter.h" | 27 #include "SkDrawFilter.h" |
28 #include "SkGPipe.h" | 28 #include "SkGPipe.h" |
29 #include "SkGraphics.h" | 29 #include "SkGraphics.h" |
30 #include "SkImageDecoder.h" | 30 #include "SkImageDecoder.h" |
31 #include "SkImageEncoder.h" | 31 #include "SkImageEncoder.h" |
(...skipping 362 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
394 * | 394 * |
395 * TODO(epoger): Come up with a better solution that allows us to | 395 * TODO(epoger): Come up with a better solution that allows us to |
396 * compare full pixel data, including alpha channel, while still being | 396 * compare full pixel data, including alpha channel, while still being |
397 * robust in the face of transformations to/from PNG files. | 397 * robust in the face of transformations to/from PNG files. |
398 * Options include: | 398 * Options include: |
399 * | 399 * |
400 * 1. Continue to call force_all_opaque(), but ONLY for bitmaps that | 400 * 1. Continue to call force_all_opaque(), but ONLY for bitmaps that |
401 * will be written to, or compared against, PNG files. | 401 * will be written to, or compared against, PNG files. |
402 * PRO: Preserve/compare alpha channel info for the non-PNG cases | 402 * PRO: Preserve/compare alpha channel info for the non-PNG cases |
403 * (comparing different renderModes in-memory) | 403 * (comparing different renderModes in-memory) |
404 * CON: The bitmaps (and checksums) for these non-PNG cases would be | 404 * CON: The bitmaps (and hash digests) for these non-PNG cases would be |
405 * different than those for the PNG-compared cases, and in the | 405 * different than those for the PNG-compared cases, and in the |
406 * case of a failed renderMode comparison, how would we write the | 406 * case of a failed renderMode comparison, how would we write the |
407 * image to disk for examination? | 407 * image to disk for examination? |
408 * | 408 * |
409 * 2. Always compute image checksums from PNG format (either | 409 * 2. Always compute image hash digests from PNG format (either |
410 * directly from the the bytes of a PNG file, or capturing the | 410 * directly from the the bytes of a PNG file, or capturing the |
411 * bytes we would have written to disk if we were writing the | 411 * bytes we would have written to disk if we were writing the |
412 * bitmap out as a PNG). | 412 * bitmap out as a PNG). |
413 * PRO: I think this would allow us to never force opaque, and to | 413 * PRO: I think this would allow us to never force opaque, and to |
414 * the extent that alpha channel data can be preserved in a PNG | 414 * the extent that alpha channel data can be preserved in a PNG |
415 * file, we could observe it. | 415 * file, we could observe it. |
416 * CON: If we read a bitmap from disk, we need to take its checksum | 416 * CON: If we read a bitmap from disk, we need to take its hash digest |
417 * from the source PNG (we can't compute it from the bitmap we | 417 * from the source PNG (we can't compute it from the bitmap we |
418 * read out of the PNG, because we will have already premultiplied | 418 * read out of the PNG, because we will have already premultiplied |
419 * the alpha). | 419 * the alpha). |
420 * CON: Seems wasteful to convert a bitmap to PNG format just to take | 420 * CON: Seems wasteful to convert a bitmap to PNG format just to take |
421 * its checksum. (Although we're wasting lots of effort already | 421 * its hash digest. (Although we're wasting lots of effort already |
422 * calling force_all_opaque().) | 422 * calling force_all_opaque().) |
423 * | 423 * |
424 * 3. Make the alpha premultiply/unpremultiply routines 100% consistent, | 424 * 3. Make the alpha premultiply/unpremultiply routines 100% consistent, |
425 * so we can transform images back and forth without fear of off-by-one | 425 * so we can transform images back and forth without fear of off-by-one |
426 * errors. | 426 * errors. |
427 * CON: Math is hard. | 427 * CON: Math is hard. |
428 * | 428 * |
429 * 4. Perform a "close enough" comparison of bitmaps (+/- 1 bit in each | 429 * 4. Perform a "close enough" comparison of bitmaps (+/- 1 bit in each |
430 * channel), rather than demanding absolute equality. | 430 * channel), rather than demanding absolute equality. |
431 * CON: Can't do this with checksums. | 431 * CON: Can't do this with hash digests. |
432 */ | 432 */ |
433 static void complete_bitmap(SkBitmap* bitmap) { | 433 static void complete_bitmap(SkBitmap* bitmap) { |
434 force_all_opaque(*bitmap); | 434 force_all_opaque(*bitmap); |
435 } | 435 } |
436 | 436 |
437 static void installFilter(SkCanvas* canvas); | 437 static void installFilter(SkCanvas* canvas); |
438 | 438 |
439 static void invokeGM(GM* gm, SkCanvas* canvas, bool isPDF, bool isDeferred)
{ | 439 static void invokeGM(GM* gm, SkCanvas* canvas, bool isPDF, bool isDeferred)
{ |
440 SkAutoCanvasRestore acr(canvas, true); | 440 SkAutoCanvasRestore acr(canvas, true); |
441 | 441 |
(...skipping 241 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
683 (int)SkGetPackedA32(actualPixel
))); | 683 (int)SkGetPackedA32(actualPixel
))); |
684 } | 684 } |
685 } | 685 } |
686 } | 686 } |
687 gm_fprintf(stderr, "---- %s: %d (of %d) differing pixels," | 687 gm_fprintf(stderr, "---- %s: %d (of %d) differing pixels," |
688 " max per-channel mismatch R=%d G=%d B=%d A=%d\n", | 688 " max per-channel mismatch R=%d G=%d B=%d A=%d\n", |
689 testName, differingPixels, width*height, errR, errG, errB, er
rA); | 689 testName, differingPixels, width*height, errR, errG, errB, er
rA); |
690 } | 690 } |
691 | 691 |
692 /** | 692 /** |
693 * Compares actual checksum to expectations, returning the set of errors | 693 * Compares actual hash digest to expectations, returning the set of errors |
694 * (if any) that we saw along the way. | 694 * (if any) that we saw along the way. |
695 * | 695 * |
696 * If fMismatchPath has been set, and there are pixel diffs, then the | 696 * If fMismatchPath has been set, and there are pixel diffs, then the |
697 * actual bitmap will be written out to a file within fMismatchPath. | 697 * actual bitmap will be written out to a file within fMismatchPath. |
698 * | 698 * |
699 * @param expectations what expectations to compare actualBitmap against | 699 * @param expectations what expectations to compare actualBitmap against |
700 * @param actualBitmap the image we actually generated | 700 * @param actualBitmap the image we actually generated |
701 * @param baseNameString name of test without renderModeDescriptor added | 701 * @param baseNameString name of test without renderModeDescriptor added |
702 * @param renderModeDescriptor e.g., "-rtree", "-deferred" | 702 * @param renderModeDescriptor e.g., "-rtree", "-deferred" |
703 * @param addToJsonSummary whether to add these results (both actual and | 703 * @param addToJsonSummary whether to add these results (both actual and |
704 * expected) to the JSON summary. Regardless of this setting, if | 704 * expected) to the JSON summary. Regardless of this setting, if |
705 * we find an image mismatch in this test, we will write these | 705 * we find an image mismatch in this test, we will write these |
706 * results to the JSON summary. (This is so that we will always | 706 * results to the JSON summary. (This is so that we will always |
707 * report errors across rendering modes, such as pipe vs tiled. | 707 * report errors across rendering modes, such as pipe vs tiled. |
708 * See https://codereview.chromium.org/13650002/ ) | 708 * See https://codereview.chromium.org/13650002/ ) |
709 */ | 709 */ |
710 ErrorCombination compare_to_expectations(Expectations expectations, | 710 ErrorCombination compare_to_expectations(Expectations expectations, |
711 const SkBitmap& actualBitmap, | 711 const SkBitmap& actualBitmap, |
712 const SkString& baseNameString, | 712 const SkString& baseNameString, |
713 const char renderModeDescriptor[], | 713 const char renderModeDescriptor[], |
714 bool addToJsonSummary) { | 714 bool addToJsonSummary) { |
715 ErrorCombination errors; | 715 ErrorCombination errors; |
716 Checksum actualChecksum = SkBitmapChecksummer::Compute64(actualBitmap); | 716 SkHashDigest actualBitmapHash; |
| 717 // TODO(epoger): Better handling for error returned by ComputeDigest()? |
| 718 // For now, we just report a digest of 0 in error cases, like before. |
| 719 if (!SkBitmapHasher::ComputeDigest(actualBitmap, &actualBitmapHash)) { |
| 720 actualBitmapHash = 0; |
| 721 } |
717 SkString completeNameString = baseNameString; | 722 SkString completeNameString = baseNameString; |
718 completeNameString.append(renderModeDescriptor); | 723 completeNameString.append(renderModeDescriptor); |
719 const char* completeName = completeNameString.c_str(); | 724 const char* completeName = completeNameString.c_str(); |
720 | 725 |
721 if (expectations.empty()) { | 726 if (expectations.empty()) { |
722 errors.add(kMissingExpectations_ErrorType); | 727 errors.add(kMissingExpectations_ErrorType); |
723 } else if (!expectations.match(actualChecksum)) { | 728 } else if (!expectations.match(actualBitmapHash)) { |
724 addToJsonSummary = true; | 729 addToJsonSummary = true; |
725 // The error mode we record depends on whether this was running | 730 // The error mode we record depends on whether this was running |
726 // in a non-standard renderMode. | 731 // in a non-standard renderMode. |
727 if ('\0' == *renderModeDescriptor) { | 732 if ('\0' == *renderModeDescriptor) { |
728 errors.add(kExpectationsMismatch_ErrorType); | 733 errors.add(kExpectationsMismatch_ErrorType); |
729 } else { | 734 } else { |
730 errors.add(kRenderModeMismatch_ErrorType); | 735 errors.add(kRenderModeMismatch_ErrorType); |
731 } | 736 } |
732 | 737 |
733 // Write out the "actuals" for any mismatches, if we have | 738 // Write out the "actuals" for any mismatches, if we have |
734 // been directed to do so. | 739 // been directed to do so. |
735 if (fMismatchPath) { | 740 if (fMismatchPath) { |
736 SkString path = | 741 SkString path = |
737 make_filename(fMismatchPath, renderModeDescriptor, | 742 make_filename(fMismatchPath, renderModeDescriptor, |
738 baseNameString.c_str(), "png"); | 743 baseNameString.c_str(), "png"); |
739 write_bitmap(path, actualBitmap); | 744 write_bitmap(path, actualBitmap); |
740 } | 745 } |
741 | 746 |
742 // If we have access to a single expected bitmap, log more | 747 // If we have access to a single expected bitmap, log more |
743 // detail about the mismatch. | 748 // detail about the mismatch. |
744 const SkBitmap *expectedBitmapPtr = expectations.asBitmap(); | 749 const SkBitmap *expectedBitmapPtr = expectations.asBitmap(); |
745 if (NULL != expectedBitmapPtr) { | 750 if (NULL != expectedBitmapPtr) { |
746 report_bitmap_diffs(*expectedBitmapPtr, actualBitmap, completeNa
me); | 751 report_bitmap_diffs(*expectedBitmapPtr, actualBitmap, completeNa
me); |
747 } | 752 } |
748 } | 753 } |
749 RecordTestResults(errors, baseNameString, renderModeDescriptor); | 754 RecordTestResults(errors, baseNameString, renderModeDescriptor); |
750 | 755 |
751 if (addToJsonSummary) { | 756 if (addToJsonSummary) { |
752 add_actual_results_to_json_summary(completeName, actualChecksum, err
ors, | 757 add_actual_results_to_json_summary(completeName, actualBitmapHash, e
rrors, |
753 expectations.ignoreFailure()); | 758 expectations.ignoreFailure()); |
754 add_expected_results_to_json_summary(completeName, expectations); | 759 add_expected_results_to_json_summary(completeName, expectations); |
755 } | 760 } |
756 | 761 |
757 return errors; | 762 return errors; |
758 } | 763 } |
759 | 764 |
760 /** | 765 /** |
761 * Add this result to the appropriate JSON collection of actual results, | 766 * Add this result to the appropriate JSON collection of actual results, |
762 * depending on status. | 767 * depending on status. |
763 */ | 768 */ |
764 void add_actual_results_to_json_summary(const char testName[], | 769 void add_actual_results_to_json_summary(const char testName[], |
765 Checksum actualChecksum, | 770 const SkHashDigest& actualBitmapHash
, |
766 ErrorCombination result, | 771 ErrorCombination result, |
767 bool ignoreFailure) { | 772 bool ignoreFailure) { |
768 Json::Value actualResults; | 773 Json::Value actualResults; |
769 actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] = | 774 actualResults[kJsonKey_ActualResults_AnyStatus_Checksum] = |
770 asJsonValue(actualChecksum); | 775 asJsonValue(actualBitmapHash); |
771 if (result.isEmpty()) { | 776 if (result.isEmpty()) { |
772 this->fJsonActualResults_Succeeded[testName] = actualResults; | 777 this->fJsonActualResults_Succeeded[testName] = actualResults; |
773 } else { | 778 } else { |
774 if (ignoreFailure) { | 779 if (ignoreFailure) { |
775 // TODO: Once we have added the ability to compare | 780 // TODO: Once we have added the ability to compare |
776 // actual results against expectations in a JSON file | 781 // actual results against expectations in a JSON file |
777 // (where we can set ignore-failure to either true or | 782 // (where we can set ignore-failure to either true or |
778 // false), add test cases that exercise ignored | 783 // false), add test cases that exercise ignored |
779 // failures (both for kMissingExpectations_ErrorType | 784 // failures (both for kMissingExpectations_ErrorType |
780 // and kExpectationsMismatch_ErrorType). | 785 // and kExpectationsMismatch_ErrorType). |
781 this->fJsonActualResults_FailureIgnored[testName] = | 786 this->fJsonActualResults_FailureIgnored[testName] = |
782 actualResults; | 787 actualResults; |
783 } else { | 788 } else { |
784 if (result.includes(kMissingExpectations_ErrorType)) { | 789 if (result.includes(kMissingExpectations_ErrorType)) { |
785 // TODO: What about the case where there IS an | 790 // TODO: What about the case where there IS an |
786 // expected image checksum, but that gm test | 791 // expected image hash digest, but that gm test |
787 // doesn't actually run? For now, those cases | 792 // doesn't actually run? For now, those cases |
788 // will always be ignored, because gm only looks | 793 // will always be ignored, because gm only looks |
789 // at expectations that correspond to gm tests | 794 // at expectations that correspond to gm tests |
790 // that were actually run. | 795 // that were actually run. |
791 // | 796 // |
792 // Once we have the ability to express | 797 // Once we have the ability to express |
793 // expectations as a JSON file, we should fix this | 798 // expectations as a JSON file, we should fix this |
794 // (and add a test case for which an expectation | 799 // (and add a test case for which an expectation |
795 // is given but the test is never run). | 800 // is given but the test is never run). |
796 this->fJsonActualResults_NoComparison[testName] = | 801 this->fJsonActualResults_NoComparison[testName] = |
(...skipping 37 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
834 GM* gm, const ConfigData& gRec, const char writePath[], | 839 GM* gm, const ConfigData& gRec, const char writePath[], |
835 SkBitmap& actualBitmap, SkDynamicMemoryWStream* pdf) { | 840 SkBitmap& actualBitmap, SkDynamicMemoryWStream* pdf) { |
836 | 841 |
837 SkString name = make_name(gm->shortName(), gRec.fName); | 842 SkString name = make_name(gm->shortName(), gRec.fName); |
838 ErrorCombination errors; | 843 ErrorCombination errors; |
839 | 844 |
840 ExpectationsSource *expectationsSource = this->fExpectationsSource.get()
; | 845 ExpectationsSource *expectationsSource = this->fExpectationsSource.get()
; |
841 if (expectationsSource && (gRec.fFlags & kRead_ConfigFlag)) { | 846 if (expectationsSource && (gRec.fFlags & kRead_ConfigFlag)) { |
842 /* | 847 /* |
843 * Get the expected results for this test, as one or more allowed | 848 * Get the expected results for this test, as one or more allowed |
844 * checksums. The current implementation of expectationsSource | 849 * hash digests. The current implementation of expectationsSource |
845 * get this by computing the checksum of a single PNG file on disk. | 850 * get this by computing the hash digest of a single PNG file on dis
k. |
846 * | 851 * |
847 * TODO(epoger): This relies on the fact that | 852 * TODO(epoger): This relies on the fact that |
848 * force_all_opaque() was called on the bitmap before it | 853 * force_all_opaque() was called on the bitmap before it |
849 * was written to disk as a PNG in the first place. If | 854 * was written to disk as a PNG in the first place. If |
850 * not, the checksum returned here may not match the | 855 * not, the hash digest returned here may not match the |
851 * checksum of actualBitmap, which *has* been run through | 856 * hash digest of actualBitmap, which *has* been run through |
852 * force_all_opaque(). | 857 * force_all_opaque(). |
853 * See comments above complete_bitmap() for more detail. | 858 * See comments above complete_bitmap() for more detail. |
854 */ | 859 */ |
855 Expectations expectations = expectationsSource->get(name.c_str()); | 860 Expectations expectations = expectationsSource->get(name.c_str()); |
856 errors.add(compare_to_expectations(expectations, actualBitmap, | 861 errors.add(compare_to_expectations(expectations, actualBitmap, |
857 name, "", true)); | 862 name, "", true)); |
858 } else { | 863 } else { |
859 // If we are running without expectations, we still want to | 864 // If we are running without expectations, we still want to |
860 // record the actual results. | 865 // record the actual results. |
861 Checksum actualChecksum = | 866 SkHashDigest actualBitmapHash; |
862 SkBitmapChecksummer::Compute64(actualBitmap); | 867 // TODO(epoger): Better handling for error returned by ComputeDigest
()? |
863 add_actual_results_to_json_summary(name.c_str(), actualChecksum, | 868 // For now, we just report a digest of 0 in error cases, like before
. |
| 869 if (!SkBitmapHasher::ComputeDigest(actualBitmap, &actualBitmapHash))
{ |
| 870 actualBitmapHash = 0; |
| 871 } |
| 872 add_actual_results_to_json_summary(name.c_str(), actualBitmapHash, |
864 ErrorCombination(kMissingExpectat
ions_ErrorType), | 873 ErrorCombination(kMissingExpectat
ions_ErrorType), |
865 false); | 874 false); |
866 RecordTestResults(ErrorCombination(kMissingExpectations_ErrorType),
name, ""); | 875 RecordTestResults(ErrorCombination(kMissingExpectations_ErrorType),
name, ""); |
867 } | 876 } |
868 | 877 |
869 // TODO: Consider moving this into compare_to_expectations(), | 878 // TODO: Consider moving this into compare_to_expectations(), |
870 // similar to fMismatchPath... for now, we don't do that, because | 879 // similar to fMismatchPath... for now, we don't do that, because |
871 // we don't want to write out the actual bitmaps for all | 880 // we don't want to write out the actual bitmaps for all |
872 // renderModes of all tests! That would be a lot of files. | 881 // renderModes of all tests! That would be a lot of files. |
873 if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) { | 882 if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) { |
(...skipping 224 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1098 bool fUseFileHierarchy; | 1107 bool fUseFileHierarchy; |
1099 ErrorCombination fIgnorableErrorCombination; | 1108 ErrorCombination fIgnorableErrorCombination; |
1100 | 1109 |
1101 const char* fMismatchPath; | 1110 const char* fMismatchPath; |
1102 | 1111 |
1103 // collection of tests that have failed with each ErrorType | 1112 // collection of tests that have failed with each ErrorType |
1104 SkTArray<SkString> fFailedTests[kLast_ErrorType+1]; | 1113 SkTArray<SkString> fFailedTests[kLast_ErrorType+1]; |
1105 int fTestsRun; | 1114 int fTestsRun; |
1106 SkTDict<int> fRenderModesEncountered; | 1115 SkTDict<int> fRenderModesEncountered; |
1107 | 1116 |
1108 // Where to read expectations (expected image checksums, etc.) from. | 1117 // Where to read expectations (expected image hash digests, etc.) from. |
1109 // If unset, we don't do comparisons. | 1118 // If unset, we don't do comparisons. |
1110 SkAutoTUnref<ExpectationsSource> fExpectationsSource; | 1119 SkAutoTUnref<ExpectationsSource> fExpectationsSource; |
1111 | 1120 |
1112 // JSON summaries that we generate as we go (just for output). | 1121 // JSON summaries that we generate as we go (just for output). |
1113 Json::Value fJsonExpectedResults; | 1122 Json::Value fJsonExpectedResults; |
1114 Json::Value fJsonActualResults_Failed; | 1123 Json::Value fJsonActualResults_Failed; |
1115 Json::Value fJsonActualResults_FailureIgnored; | 1124 Json::Value fJsonActualResults_FailureIgnored; |
1116 Json::Value fJsonActualResults_NoComparison; | 1125 Json::Value fJsonActualResults_NoComparison; |
1117 Json::Value fJsonActualResults_Succeeded; | 1126 Json::Value fJsonActualResults_Succeeded; |
1118 | 1127 |
(...skipping 759 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
1878 if (FLAGS_forceBWtext) { | 1887 if (FLAGS_forceBWtext) { |
1879 canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref(); | 1888 canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref(); |
1880 } | 1889 } |
1881 } | 1890 } |
1882 | 1891 |
1883 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL) | 1892 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL) |
1884 int main(int argc, char * const argv[]) { | 1893 int main(int argc, char * const argv[]) { |
1885 return tool_main(argc, (char**) argv); | 1894 return tool_main(argc, (char**) argv); |
1886 } | 1895 } |
1887 #endif | 1896 #endif |
OLD | NEW |