Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(436)

Side by Side Diff: gm/gmmain.cpp

Issue 12851022: gm: display summary of all results by ErrorType (Closed) Base URL: http://skia.googlecode.com/svn/trunk/
Patch Set: synced_to_r8509 Created 7 years, 8 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch | Annotate | Revision Log
OLDNEW
1 /* 1 /*
2 * Copyright 2011 Google Inc. 2 * Copyright 2011 Google Inc.
3 * 3 *
4 * Use of this source code is governed by a BSD-style license that can be 4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file. 5 * found in the LICENSE file.
6 */ 6 */
7 7
8 /* 8 /*
9 * Code for the "gm" (Golden Master) rendering comparison tool. 9 * Code for the "gm" (Golden Master) rendering comparison tool.
10 * 10 *
(...skipping 16 matching lines...) Expand all
27 #include "SkDrawFilter.h" 27 #include "SkDrawFilter.h"
28 #include "SkGPipe.h" 28 #include "SkGPipe.h"
29 #include "SkGraphics.h" 29 #include "SkGraphics.h"
30 #include "SkImageDecoder.h" 30 #include "SkImageDecoder.h"
31 #include "SkImageEncoder.h" 31 #include "SkImageEncoder.h"
32 #include "SkOSFile.h" 32 #include "SkOSFile.h"
33 #include "SkPicture.h" 33 #include "SkPicture.h"
34 #include "SkRefCnt.h" 34 #include "SkRefCnt.h"
35 #include "SkStream.h" 35 #include "SkStream.h"
36 #include "SkTArray.h" 36 #include "SkTArray.h"
37 #include "SkTDict.h"
37 #include "SkTileGridPicture.h" 38 #include "SkTileGridPicture.h"
38 #include "SamplePipeControllers.h" 39 #include "SamplePipeControllers.h"
39 40
40 #ifdef SK_BUILD_FOR_WIN 41 #ifdef SK_BUILD_FOR_WIN
41 // json includes xlocale which generates warning 4530 because we're compilin g without 42 // json includes xlocale which generates warning 4530 because we're compilin g without
42 // exceptions; see https://code.google.com/p/skia/issues/detail?id=1067 43 // exceptions; see https://code.google.com/p/skia/issues/detail?id=1067
43 #pragma warning(push) 44 #pragma warning(push)
44 #pragma warning(disable : 4530) 45 #pragma warning(disable : 4530)
45 #endif 46 #endif
46 #include "json/value.h" 47 #include "json/value.h"
(...skipping 31 matching lines...) Expand 10 before | Expand all | Expand 10 after
78 79
79 #ifdef SK_BUILD_FOR_MAC 80 #ifdef SK_BUILD_FOR_MAC
80 #include "SkCGUtils.h" 81 #include "SkCGUtils.h"
81 #define CAN_IMAGE_PDF 1 82 #define CAN_IMAGE_PDF 1
82 #else 83 #else
83 #define CAN_IMAGE_PDF 0 84 #define CAN_IMAGE_PDF 0
84 #endif 85 #endif
85 86
86 using namespace skiagm; 87 using namespace skiagm;
87 88
88 struct FailRec {
89 SkString fName;
90 bool fIsPixelError;
91
92 FailRec() : fIsPixelError(false) {}
93 FailRec(const SkString& name) : fName(name), fIsPixelError(false) {}
94 };
95
96 class Iter { 89 class Iter {
97 public: 90 public:
98 Iter() { 91 Iter() {
99 this->reset(); 92 this->reset();
100 } 93 }
101 94
102 void reset() { 95 void reset() {
103 fReg = GMRegistry::Head(); 96 fReg = GMRegistry::Head();
104 } 97 }
105 98
(...skipping 70 matching lines...) Expand 10 before | Expand all | Expand 10 after
176 169
177 static PipeFlagComboData gPipeWritingFlagCombos[] = { 170 static PipeFlagComboData gPipeWritingFlagCombos[] = {
178 { "", 0 }, 171 { "", 0 },
179 { " cross-process", SkGPipeWriter::kCrossProcess_Flag }, 172 { " cross-process", SkGPipeWriter::kCrossProcess_Flag },
180 { " cross-process, shared address", SkGPipeWriter::kCrossProcess_Flag 173 { " cross-process, shared address", SkGPipeWriter::kCrossProcess_Flag
181 | SkGPipeWriter::kSharedAddressSpace_Flag } 174 | SkGPipeWriter::kSharedAddressSpace_Flag }
182 }; 175 };
183 176
184 class GMMain { 177 class GMMain {
185 public: 178 public:
186 GMMain() { 179 GMMain() : fUseFileHierarchy(false), fMismatchPath(NULL), fTestsRun(0),
187 // Set default values of member variables, which tool_main() 180 fRenderModesEncountered(1) {
188 // may override.
189 fUseFileHierarchy = false;
190 fIgnorableErrorCombination.add(kMissingExpectations_ErrorType); 181 fIgnorableErrorCombination.add(kMissingExpectations_ErrorType);
191 fMismatchPath = NULL;
192 } 182 }
193 183
194 SkString make_name(const char shortName[], const char configName[]) { 184 SkString make_name(const char shortName[], const char configName[]) {
195 SkString name; 185 SkString name;
196 if (0 == strlen(configName)) { 186 if (0 == strlen(configName)) {
197 name.append(shortName); 187 name.append(shortName);
198 } else if (fUseFileHierarchy) { 188 } else if (fUseFileHierarchy) {
199 name.appendf("%s%c%s", configName, SkPATH_SEPARATOR, shortName); 189 name.appendf("%s%c%s", configName, SkPATH_SEPARATOR, shortName);
200 } else { 190 } else {
201 name.appendf("%s_%s", shortName, configName); 191 name.appendf("%s_%s", shortName, configName);
(...skipping 33 matching lines...) Expand 10 before | Expand all | Expand 10 after
235 // TODO(epoger): Now that we have removed force_all_opaque() 225 // TODO(epoger): Now that we have removed force_all_opaque()
236 // from this method, we should be able to get rid of the 226 // from this method, we should be able to get rid of the
237 // transformation to 8888 format also. 227 // transformation to 8888 format also.
238 SkBitmap copy; 228 SkBitmap copy;
239 bitmap.copyTo(&copy, SkBitmap::kARGB_8888_Config); 229 bitmap.copyTo(&copy, SkBitmap::kARGB_8888_Config);
240 return SkImageEncoder::EncodeFile(path.c_str(), copy, 230 return SkImageEncoder::EncodeFile(path.c_str(), copy,
241 SkImageEncoder::kPNG_Type, 100); 231 SkImageEncoder::kPNG_Type, 100);
242 } 232 }
243 233
244 /** 234 /**
245 * Records the errors encountered in fFailedTests, except for any error 235 * Add all render modes encountered thus far to the "modes" array.
246 * types we want to ignore.
247 */ 236 */
248 void RecordError(const ErrorCombination& errorCombination, const SkString& n ame, 237 void GetRenderModesEncountered(SkTArray<SkString> &modes) {
249 const char renderModeDescriptor []) { 238 SkTDict<int>::Iter iter(this->fRenderModesEncountered);
250 // The common case: no error means nothing to record. 239 const char* mode;
240 while ((mode = iter.next(NULL)) != NULL) {
241 SkString modeAsString = SkString(mode);
242 // TODO(epoger): It seems a bit silly that all of these modes were
243 // recorded with a leading "-" which we have to remove here
244 // (except for mode "", which means plain old original mode).
245 // But that's how renderModeDescriptor has been passed into
246 // compare_test_results_to_reference_bitmap() historically,
247 // and changing that now may affect other parts of our code.
248 if (modeAsString.startsWith("-")) {
249 modeAsString.remove(0, 1);
250 modes.push_back(modeAsString);
251 }
252 }
253 }
254
255 /**
256 * Records the results of this test in fTestsRun and fFailedTests.
257 *
258 * We even record successes, and errors that we regard as
259 * "ignorable"; we can filter them out later.
260 */
261 void RecordTestResults(const ErrorCombination& errorCombination, const SkStr ing& name,
262 const char renderModeDescriptor []) {
263 // Things to do regardless of errorCombination.
264 fTestsRun++;
265 int renderModeCount = 0;
266 this->fRenderModesEncountered.find(renderModeDescriptor, &renderModeCoun t);
267 renderModeCount++;
268 this->fRenderModesEncountered.set(renderModeDescriptor, renderModeCount) ;
269
251 if (errorCombination.isEmpty()) { 270 if (errorCombination.isEmpty()) {
252 return; 271 return;
253 } 272 }
254 273
255 // If only certain error type(s) were reported, we know we can ignore th em. 274 // Things to do only if there is some error condition.
256 if (errorCombination.minus(fIgnorableErrorCombination).isEmpty()) { 275 SkString fullName = make_name(name.c_str(), renderModeDescriptor);
257 return; 276 for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
258 } 277 ErrorType type = static_cast<ErrorType>(typeInt);
259 278 if (errorCombination.includes(type)) {
260 FailRec& rec = fFailedTests.push_back(make_name(name.c_str(), renderMode Descriptor)); 279 fFailedTests[type].push_back(fullName);
261 rec.fIsPixelError = errorCombination.includes(kImageMismatch_ErrorType);
262 }
263
264 // List contents of fFailedTests via SkDebug.
265 void ListErrors() {
266 for (int i = 0; i < fFailedTests.count(); ++i) {
267 if (fFailedTests[i].fIsPixelError) {
268 gm_fprintf(stderr, "\t\t%s pixel_error\n", fFailedTests[i].fName .c_str());
269 } else {
270 gm_fprintf(stderr, "\t\t%s\n", fFailedTests[i].fName.c_str());
271 } 280 }
272 } 281 }
273 } 282 }
274 283
284 /**
285 * Return the number of significant (non-ignorable) errors we have
286 * encountered so far.
287 */
288 int NumSignificantErrors() {
289 int significantErrors = 0;
290 for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
291 ErrorType type = static_cast<ErrorType>(typeInt);
292 if (!fIgnorableErrorCombination.includes(type)) {
293 significantErrors += fFailedTests[type].count();
294 }
295 }
296 return significantErrors;
297 }
298
299 /**
300 * List contents of fFailedTests to stdout.
301 */
302 void ListErrors() {
303 // First, print a single summary line.
304 SkString summary;
305 summary.appendf("Ran %d tests:", fTestsRun);
306 for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
307 ErrorType type = static_cast<ErrorType>(typeInt);
308 summary.appendf(" %s=%d", getErrorTypeName(type), fFailedTests[type] .count());
309 }
310 gm_fprintf(stdout, "%s\n", summary.c_str());
311
312 // Now, for each failure type, list the tests that failed that way.
313 for (int typeInt = 0; typeInt <= kLast_ErrorType; typeInt++) {
314 SkString line;
315 ErrorType type = static_cast<ErrorType>(typeInt);
316 if (fIgnorableErrorCombination.includes(type)) {
317 line.append("[ ] ");
318 } else {
319 line.append("[*] ");
320 }
321
322 SkTArray<SkString> *failedTestsOfThisType = &fFailedTests[type];
323 int count = failedTestsOfThisType->count();
324 line.appendf("%d %s:", count, getErrorTypeName(type));
325 for (int i = 0; i < count; ++i) {
326 line.append(" ");
327 line.append((*failedTestsOfThisType)[i]);
328 }
329 gm_fprintf(stdout, "%s\n", line.c_str());
330 }
331 gm_fprintf(stdout, "(results marked with [*] will cause nonzero return v alue)\n");
332 }
333
275 static bool write_document(const SkString& path, 334 static bool write_document(const SkString& path,
276 const SkDynamicMemoryWStream& document) { 335 const SkDynamicMemoryWStream& document) {
277 SkFILEWStream stream(path.c_str()); 336 SkFILEWStream stream(path.c_str());
278 SkAutoDataUnref data(document.copyToData()); 337 SkAutoDataUnref data(document.copyToData());
279 return stream.writeData(data.get()); 338 return stream.writeData(data.get());
280 } 339 }
281 340
282 /** 341 /**
283 * Prepare an SkBitmap to render a GM into. 342 * Prepare an SkBitmap to render a GM into.
284 * 343 *
(...skipping 250 matching lines...) Expand 10 before | Expand all | Expand 10 after
535 if (kXPS_Backend == gRec.fBackend) { 594 if (kXPS_Backend == gRec.fBackend) {
536 path = make_filename(writePath, renderModeDescriptor, name.c_str(), 595 path = make_filename(writePath, renderModeDescriptor, name.c_str(),
537 "xps"); 596 "xps");
538 success = write_document(path, *document); 597 success = write_document(path, *document);
539 } 598 }
540 if (success) { 599 if (success) {
541 return kEmpty_ErrorCombination; 600 return kEmpty_ErrorCombination;
542 } else { 601 } else {
543 gm_fprintf(stderr, "FAILED to write %s\n", path.c_str()); 602 gm_fprintf(stderr, "FAILED to write %s\n", path.c_str());
544 ErrorCombination errors(kWritingReferenceImage_ErrorType); 603 ErrorCombination errors(kWritingReferenceImage_ErrorType);
545 RecordError(errors, name, renderModeDescriptor); 604 // TODO(epoger): Don't call RecordTestResults() here...
605 // Instead, we should make sure to call RecordTestResults
606 // exactly ONCE per test. (Otherwise, gmmain.fTestsRun
607 // will be incremented twice for this test: once in
608 // compare_test_results_to_stored_expectations() before
609 // that method calls this one, and again here.)
610 //
611 // When we make that change, we should probably add a
612 // WritingReferenceImage test to the gm self-tests.)
613 RecordTestResults(errors, name, renderModeDescriptor);
546 return errors; 614 return errors;
547 } 615 }
548 } 616 }
549 617
550 /** 618 /**
551 * Log more detail about the mistmatch between expectedBitmap and 619 * Log more detail about the mistmatch between expectedBitmap and
552 * actualBitmap. 620 * actualBitmap.
553 */ 621 */
554 void report_bitmap_diffs(const SkBitmap& expectedBitmap, const SkBitmap& act ualBitmap, 622 void report_bitmap_diffs(const SkBitmap& expectedBitmap, const SkBitmap& act ualBitmap,
555 const char *testName) { 623 const char *testName) {
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after
647 write_bitmap(path, actualBitmap); 715 write_bitmap(path, actualBitmap);
648 } 716 }
649 717
650 // If we have access to a single expected bitmap, log more 718 // If we have access to a single expected bitmap, log more
651 // detail about the mismatch. 719 // detail about the mismatch.
652 const SkBitmap *expectedBitmapPtr = expectations.asBitmap(); 720 const SkBitmap *expectedBitmapPtr = expectations.asBitmap();
653 if (NULL != expectedBitmapPtr) { 721 if (NULL != expectedBitmapPtr) {
654 report_bitmap_diffs(*expectedBitmapPtr, actualBitmap, completeNa me); 722 report_bitmap_diffs(*expectedBitmapPtr, actualBitmap, completeNa me);
655 } 723 }
656 } 724 }
657 RecordError(errors, baseNameString, renderModeDescriptor); 725 RecordTestResults(errors, baseNameString, renderModeDescriptor);
658 726
659 if (addToJsonSummary) { 727 if (addToJsonSummary) {
660 add_actual_results_to_json_summary(completeName, actualChecksum, err ors, 728 add_actual_results_to_json_summary(completeName, actualChecksum, err ors,
661 expectations.ignoreFailure()); 729 expectations.ignoreFailure());
662 add_expected_results_to_json_summary(completeName, expectations); 730 add_expected_results_to_json_summary(completeName, expectations);
663 } 731 }
664 732
665 return errors; 733 return errors;
666 } 734 }
667 735
(...skipping 95 matching lines...) Expand 10 before | Expand all | Expand 10 after
763 errors.add(compare_to_expectations(expectations, actualBitmap, 831 errors.add(compare_to_expectations(expectations, actualBitmap,
764 name, "", true)); 832 name, "", true));
765 } else { 833 } else {
766 // If we are running without expectations, we still want to 834 // If we are running without expectations, we still want to
767 // record the actual results. 835 // record the actual results.
768 Checksum actualChecksum = 836 Checksum actualChecksum =
769 SkBitmapChecksummer::Compute64(actualBitmap); 837 SkBitmapChecksummer::Compute64(actualBitmap);
770 add_actual_results_to_json_summary(name.c_str(), actualChecksum, 838 add_actual_results_to_json_summary(name.c_str(), actualChecksum,
771 ErrorCombination(kMissingExpectat ions_ErrorType), 839 ErrorCombination(kMissingExpectat ions_ErrorType),
772 false); 840 false);
841 RecordTestResults(ErrorCombination(kMissingExpectations_ErrorType), name, "");
773 } 842 }
774 843
775 // TODO: Consider moving this into compare_to_expectations(), 844 // TODO: Consider moving this into compare_to_expectations(),
776 // similar to fMismatchPath... for now, we don't do that, because 845 // similar to fMismatchPath... for now, we don't do that, because
777 // we don't want to write out the actual bitmaps for all 846 // we don't want to write out the actual bitmaps for all
778 // renderModes of all tests! That would be a lot of files. 847 // renderModes of all tests! That would be a lot of files.
779 if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) { 848 if (writePath && (gRec.fFlags & kWrite_ConfigFlag)) {
780 errors.add(write_reference_image(gRec, writePath, "", 849 errors.add(write_reference_image(gRec, writePath, "",
781 name, actualBitmap, pdf)); 850 name, actualBitmap, pdf));
782 } 851 }
783 852
784 return errors; 853 return errors;
785 } 854 }
786 855
787 /** 856 /**
788 * Compare actualBitmap to referenceBitmap. 857 * Compare actualBitmap to referenceBitmap.
789 * 858 *
790 * @param gm which test generated the bitmap 859 * @param gm which test generated the bitmap
791 * @param gRec 860 * @param gRec
792 * @param renderModeDescriptor 861 * @param renderModeDescriptor
793 * @param actualBitmap actual bitmap generated by this run 862 * @param actualBitmap actual bitmap generated by this run
794 * @param referenceBitmap bitmap we expected to be generated 863 * @param referenceBitmap bitmap we expected to be generated
795 */ 864 */
796 ErrorCombination compare_test_results_to_reference_bitmap( 865 ErrorCombination compare_test_results_to_reference_bitmap(
797 GM* gm, const ConfigData& gRec, const char renderModeDescriptor [], 866 GM* gm, const ConfigData& gRec, const char renderModeDescriptor [],
798 SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) { 867 SkBitmap& actualBitmap, const SkBitmap* referenceBitmap) {
799 868
869 // TODO(epoger): This method is run to compare results across
870 // different rendering modes (as opposed to
871 // compare_test_results_to_stored_expectations(), which
872 // compares results against expectations stored on disk). If
873 // we would like the GenerateGMs step to distinguish between
874 // those two types of mismatches, we should report image
875 // mismatches in here with a different ErrorType.
800 SkASSERT(referenceBitmap); 876 SkASSERT(referenceBitmap);
801 SkString name = make_name(gm->shortName(), gRec.fName); 877 SkString name = make_name(gm->shortName(), gRec.fName);
802 Expectations expectations(*referenceBitmap); 878 Expectations expectations(*referenceBitmap);
803 return compare_to_expectations(expectations, actualBitmap, 879 return compare_to_expectations(expectations, actualBitmap,
804 name, renderModeDescriptor); 880 name, renderModeDescriptor);
805 } 881 }
806 882
807 static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t rec ordFlags, 883 static SkPicture* generate_new_picture(GM* gm, BbhType bbhType, uint32_t rec ordFlags,
808 SkScalar scale = SK_Scalar1) { 884 SkScalar scale = SK_Scalar1) {
809 // Pictures are refcounted so must be on heap 885 // Pictures are refcounted so must be on heap
(...skipping 93 matching lines...) Expand 10 before | Expand all | Expand 10 after
903 // -deferred image, we exit early! We should fix this 979 // -deferred image, we exit early! We should fix this
904 // ASAP, because it is hiding -deferred errors... but for 980 // ASAP, because it is hiding -deferred errors... but for
905 // now, I'm leaving the logic as it is so that the 981 // now, I'm leaving the logic as it is so that the
906 // refactoring change 982 // refactoring change
907 // https://codereview.chromium.org/12992003/ is unblocked. 983 // https://codereview.chromium.org/12992003/ is unblocked.
908 // 984 //
909 // Filed as https://code.google.com/p/skia/issues/detail?id=1180 985 // Filed as https://code.google.com/p/skia/issues/detail?id=1180
910 // ('image-surface gm test is failing in "deferred" mode, 986 // ('image-surface gm test is failing in "deferred" mode,
911 // and gm is not reporting the failure') 987 // and gm is not reporting the failure')
912 if (errors.isEmpty()) { 988 if (errors.isEmpty()) {
989 // TODO(epoger): Report this as a new ErrorType,
990 // something like kImageGeneration_ErrorType?
913 return kEmpty_ErrorCombination; 991 return kEmpty_ErrorCombination;
914 } 992 }
915 return compare_test_results_to_reference_bitmap( 993 return compare_test_results_to_reference_bitmap(
916 gm, gRec, "-deferred", bitmap, &referenceBitmap); 994 gm, gRec, "-deferred", bitmap, &referenceBitmap);
917 } 995 }
918 return kEmpty_ErrorCombination; 996 return kEmpty_ErrorCombination;
919 } 997 }
920 998
921 ErrorCombination test_pipe_playback(GM* gm, 999 ErrorCombination test_pipe_playback(GM* gm,
922 const ConfigData& gRec, 1000 const ConfigData& gRec,
(...skipping 53 matching lines...) Expand 10 before | Expand all | Expand 10 after
976 // 1054 //
977 // member variables. 1055 // member variables.
978 // They are public for now, to allow easier setting by tool_main(). 1056 // They are public for now, to allow easier setting by tool_main().
979 // 1057 //
980 1058
981 bool fUseFileHierarchy; 1059 bool fUseFileHierarchy;
982 ErrorCombination fIgnorableErrorCombination; 1060 ErrorCombination fIgnorableErrorCombination;
983 1061
984 const char* fMismatchPath; 1062 const char* fMismatchPath;
985 1063
986 // information about all failed tests we have encountered so far 1064 // collection of tests that have failed with each ErrorType
987 SkTArray<FailRec> fFailedTests; 1065 SkTArray<SkString> fFailedTests[kLast_ErrorType+1];
1066 int fTestsRun;
1067 SkTDict<int> fRenderModesEncountered;
988 1068
989 // Where to read expectations (expected image checksums, etc.) from. 1069 // Where to read expectations (expected image checksums, etc.) from.
990 // If unset, we don't do comparisons. 1070 // If unset, we don't do comparisons.
991 SkAutoTUnref<ExpectationsSource> fExpectationsSource; 1071 SkAutoTUnref<ExpectationsSource> fExpectationsSource;
992 1072
993 // JSON summaries that we generate as we go (just for output). 1073 // JSON summaries that we generate as we go (just for output).
994 Json::Value fJsonExpectedResults; 1074 Json::Value fJsonExpectedResults;
995 Json::Value fJsonActualResults_Failed; 1075 Json::Value fJsonActualResults_Failed;
996 Json::Value fJsonActualResults_FailureIgnored; 1076 Json::Value fJsonActualResults_FailureIgnored;
997 Json::Value fJsonActualResults_NoComparison; 1077 Json::Value fJsonActualResults_NoComparison;
(...skipping 287 matching lines...) Expand 10 before | Expand all | Expand 10 after
1285 */ 1365 */
1286 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co mpareConfig, 1366 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co mpareConfig,
1287 const SkBitmap &comparisonBitmap, 1367 const SkBitmap &comparisonBitmap,
1288 const SkTDArray<SkScalar> &tileGridReplaySca les); 1368 const SkTDArray<SkScalar> &tileGridReplaySca les);
1289 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co mpareConfig, 1369 ErrorCombination run_multiple_modes(GMMain &gmmain, GM *gm, const ConfigData &co mpareConfig,
1290 const SkBitmap &comparisonBitmap, 1370 const SkBitmap &comparisonBitmap,
1291 const SkTDArray<SkScalar> &tileGridReplaySca les) { 1371 const SkTDArray<SkScalar> &tileGridReplaySca les) {
1292 ErrorCombination errorsForAllModes; 1372 ErrorCombination errorsForAllModes;
1293 uint32_t gmFlags = gm->getFlags(); 1373 uint32_t gmFlags = gm->getFlags();
1294 1374
1295 // run the picture centric GM steps 1375 // TODO(epoger): We should start recording any per-GM skipped
1376 // modes (i.e. those we skipped due to gmFlags) with a new
1377 // ErrorType, perhaps named kIntentionallySkipped_ErrorType.
1296 if (!(gmFlags & GM::kSkipPicture_Flag)) { 1378 if (!(gmFlags & GM::kSkipPicture_Flag)) {
1297 1379
1298 ErrorCombination pictErrors; 1380 ErrorCombination pictErrors;
1299 1381
1300 //SkAutoTUnref<SkPicture> pict(generate_new_picture(gm)); 1382 //SkAutoTUnref<SkPicture> pict(generate_new_picture(gm));
1301 SkPicture* pict = gmmain.generate_new_picture(gm, kNone_BbhType, 0); 1383 SkPicture* pict = gmmain.generate_new_picture(gm, kNone_BbhType, 0);
1302 SkAutoUnref aur(pict); 1384 SkAutoUnref aur(pict);
1303 1385
1304 if (FLAGS_replay) { 1386 if (FLAGS_replay) {
1305 SkBitmap bitmap; 1387 SkBitmap bitmap;
(...skipping 77 matching lines...) Expand 10 before | Expand all | Expand 10 after
1383 if ((pipeErrors.isEmpty()) && 1465 if ((pipeErrors.isEmpty()) &&
1384 FLAGS_tiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) { 1466 FLAGS_tiledPipe && !(gmFlags & GM::kSkipTiled_Flag)) {
1385 pipeErrors.add(gmmain.test_tiled_pipe_playback(gm, compareConfig, co mparisonBitmap)); 1467 pipeErrors.add(gmmain.test_tiled_pipe_playback(gm, compareConfig, co mparisonBitmap));
1386 } 1468 }
1387 1469
1388 errorsForAllModes.add(pipeErrors); 1470 errorsForAllModes.add(pipeErrors);
1389 } 1471 }
1390 return errorsForAllModes; 1472 return errorsForAllModes;
1391 } 1473 }
1392 1474
1475 /**
1476 * Return a list of all entries in an array of strings as a single string
1477 * of this form:
1478 * "item1", "item2", "item3"
1479 */
1480 SkString list_all(const SkTArray<SkString> &stringArray);
1481 SkString list_all(const SkTArray<SkString> &stringArray) {
1482 SkString total;
1483 for (int i = 0; i < stringArray.count(); i++) {
1484 if (i > 0) {
1485 total.append(", ");
1486 }
1487 total.append("\"");
1488 total.append(stringArray[i]);
1489 total.append("\"");
1490 }
1491 return total;
1492 }
1493
1494 /**
1495 * Return a list of configuration names, as a single string of this form:
1496 * "item1", "item2", "item3"
1497 *
1498 * @param configs configurations, as a list of indices into gRec
1499 */
1500 SkString list_all_config_names(const SkTDArray<size_t> &configs);
1501 SkString list_all_config_names(const SkTDArray<size_t> &configs) {
1502 SkString total;
1503 for (int i = 0; i < configs.count(); i++) {
1504 if (i > 0) {
1505 total.append(", ");
1506 }
1507 total.append("\"");
1508 total.append(gRec[configs[i]].fName);
1509 total.append("\"");
1510 }
1511 return total;
1512 }
1513
1393 int tool_main(int argc, char** argv); 1514 int tool_main(int argc, char** argv);
1394 int tool_main(int argc, char** argv) { 1515 int tool_main(int argc, char** argv) {
1395 1516
1396 #if SK_ENABLE_INST_COUNT 1517 #if SK_ENABLE_INST_COUNT
1397 gPrintInstCount = true; 1518 gPrintInstCount = true;
1398 #endif 1519 #endif
1399 1520
1400 SkGraphics::Init(); 1521 SkGraphics::Init();
1401 // we don't need to see this during a run 1522 // we don't need to see this during a run
1402 gSkSuppressFontCachePurgeSpew = true; 1523 gSkSuppressFontCachePurgeSpew = true;
(...skipping 161 matching lines...) Expand 10 before | Expand all | Expand 10 after
1564 gm_fprintf(stderr, "reading resources from %s\n", FLAGS_resourcePath[0]) ; 1685 gm_fprintf(stderr, "reading resources from %s\n", FLAGS_resourcePath[0]) ;
1565 } 1686 }
1566 1687
1567 if (moduloDivisor <= 0) { 1688 if (moduloDivisor <= 0) {
1568 moduloRemainder = -1; 1689 moduloRemainder = -1;
1569 } 1690 }
1570 if (moduloRemainder < 0 || moduloRemainder >= moduloDivisor) { 1691 if (moduloRemainder < 0 || moduloRemainder >= moduloDivisor) {
1571 moduloRemainder = -1; 1692 moduloRemainder = -1;
1572 } 1693 }
1573 1694
1574 // Accumulate success of all tests. 1695 int gmsRun = 0;
1575 int testsRun = 0;
1576 int testsPassed = 0;
1577 int testsFailed = 0;
1578 int testsMissingReferenceImages = 0;
1579
1580 int gmIndex = -1; 1696 int gmIndex = -1;
1581 SkString moduloStr; 1697 SkString moduloStr;
1582 1698
1583 // If we will be writing out files, prepare subdirectories. 1699 // If we will be writing out files, prepare subdirectories.
1584 if (FLAGS_writePath.count() == 1) { 1700 if (FLAGS_writePath.count() == 1) {
1585 if (!sk_mkdir(FLAGS_writePath[0])) { 1701 if (!sk_mkdir(FLAGS_writePath[0])) {
1586 return -1; 1702 return -1;
1587 } 1703 }
1588 if (gmmain.fUseFileHierarchy) { 1704 if (gmmain.fUseFileHierarchy) {
1589 for (int i = 0; i < configs.count(); i++) { 1705 for (int i = 0; i < configs.count(); i++) {
(...skipping 19 matching lines...) Expand all
1609 } 1725 }
1610 moduloStr.printf("[%d.%d] ", gmIndex, moduloDivisor); 1726 moduloStr.printf("[%d.%d] ", gmIndex, moduloDivisor);
1611 } 1727 }
1612 1728
1613 const char* shortName = gm->shortName(); 1729 const char* shortName = gm->shortName();
1614 if (skip_name(FLAGS_match, shortName)) { 1730 if (skip_name(FLAGS_match, shortName)) {
1615 SkDELETE(gm); 1731 SkDELETE(gm);
1616 continue; 1732 continue;
1617 } 1733 }
1618 1734
1735 gmsRun++;
1619 SkISize size = gm->getISize(); 1736 SkISize size = gm->getISize();
1620 gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), short Name, 1737 gm_fprintf(stdout, "%sdrawing... %s [%d %d]\n", moduloStr.c_str(), short Name,
1621 size.width(), size.height()); 1738 size.width(), size.height());
1622 1739
1623 ErrorCombination testErrors; 1740 run_multiple_configs(gmmain, gm, configs, grFactory);
1624 testErrors.add(run_multiple_configs(gmmain, gm, configs, grFactory));
1625 1741
1626 SkBitmap comparisonBitmap; 1742 SkBitmap comparisonBitmap;
1627 const ConfigData compareConfig = 1743 const ConfigData compareConfig =
1628 { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextT ype, 0, kRW_ConfigFlag, "comparison", false }; 1744 { SkBitmap::kARGB_8888_Config, kRaster_Backend, kDontCare_GLContextT ype, 0, kRW_ConfigFlag, "comparison", false };
1629 testErrors.add(gmmain.generate_image(gm, compareConfig, NULL, &compariso nBitmap, false)); 1745 gmmain.generate_image(gm, compareConfig, NULL, &comparisonBitmap, false) ;
1630 1746
1631 // TODO(epoger): only run this if gmmain.generate_image() succeeded? 1747 // TODO(epoger): only run this if gmmain.generate_image() succeeded?
1632 // Otherwise, what are we comparing against? 1748 // Otherwise, what are we comparing against?
1633 testErrors.add(run_multiple_modes(gmmain, gm, compareConfig, comparisonB itmap, 1749 run_multiple_modes(gmmain, gm, compareConfig, comparisonBitmap, tileGrid ReplayScales);
1634 tileGridReplayScales));
1635
1636 // Update overall results.
1637 // We only tabulate the particular error types that we currently
1638 // care about (e.g., missing reference images). Later on, if we
1639 // want to also tabulate other error types, we can do so.
1640 testsRun++;
1641 if (!gmmain.fExpectationsSource.get() ||
1642 (testErrors.includes(kMissingExpectations_ErrorType))) {
1643 testsMissingReferenceImages++;
1644 }
1645 if (testErrors.minus(gmmain.fIgnorableErrorCombination).isEmpty()) {
1646 testsPassed++;
1647 } else {
1648 testsFailed++;
1649 }
1650 1750
1651 SkDELETE(gm); 1751 SkDELETE(gm);
1652 } 1752 }
1653 gm_fprintf(stdout, "Ran %d tests: %d passed, %d failed, %d missing reference images\n", 1753
1654 testsRun, testsPassed, testsFailed, testsMissingReferenceImages); 1754 SkTArray<SkString> modes;
1755 gmmain.GetRenderModesEncountered(modes);
1756
1757 // Output summary to stdout.
1758 gm_fprintf(stdout, "Ran %d GMs\n", gmsRun);
1759 gm_fprintf(stdout, "... over %2d configs [%s]\n", configs.count(),
1760 list_all_config_names(configs).c_str());
1761 gm_fprintf(stdout, "... and %2d modes [%s]\n", modes.count(), list_all(mo des).c_str());
1762 gm_fprintf(stdout, "... so there should be a total of %d tests.\n",
1763 gmsRun * (configs.count() + modes.count()));
1764
1765 // TODO(epoger): Ultimately, we should signal an error if the
1766 // expected total number of tests (displayed above) does not match
1767 // gmmain.fTestsRun. But for now, there are cases where those
1768 // numbers won't match: specifically, if some configs/modes are
1769 // skipped on a per-GM basis (due to gm->getFlags() for a specific
1770 // GM). Later on, we should record tests like that using some new
1771 // ErrorType, like kIntentionallySkipped_ErrorType. Then we could
1772 // signal an error if the totals didn't match up.
1655 gmmain.ListErrors(); 1773 gmmain.ListErrors();
1656 1774
1657 if (FLAGS_writeJsonSummaryPath.count() == 1) { 1775 if (FLAGS_writeJsonSummaryPath.count() == 1) {
1658 Json::Value actualResults; 1776 Json::Value actualResults;
1659 actualResults[kJsonKey_ActualResults_Failed] = 1777 actualResults[kJsonKey_ActualResults_Failed] =
1660 gmmain.fJsonActualResults_Failed; 1778 gmmain.fJsonActualResults_Failed;
1661 actualResults[kJsonKey_ActualResults_FailureIgnored] = 1779 actualResults[kJsonKey_ActualResults_FailureIgnored] =
1662 gmmain.fJsonActualResults_FailureIgnored; 1780 gmmain.fJsonActualResults_FailureIgnored;
1663 actualResults[kJsonKey_ActualResults_NoComparison] = 1781 actualResults[kJsonKey_ActualResults_NoComparison] =
1664 gmmain.fJsonActualResults_NoComparison; 1782 gmmain.fJsonActualResults_NoComparison;
(...skipping 19 matching lines...) Expand all
1684 gm_fprintf(stdout, "config: %s %x\n", config.fName, gr); 1802 gm_fprintf(stdout, "config: %s %x\n", config.fName, gr);
1685 gr->printCacheStats(); 1803 gr->printCacheStats();
1686 } 1804 }
1687 } 1805 }
1688 #endif 1806 #endif
1689 1807
1690 delete grFactory; 1808 delete grFactory;
1691 #endif 1809 #endif
1692 SkGraphics::Term(); 1810 SkGraphics::Term();
1693 1811
1694 return (0 == testsFailed) ? 0 : -1; 1812 return (0 == gmmain.NumSignificantErrors()) ? 0 : -1;
1695 } 1813 }
1696 1814
1697 void GMMain::installFilter(SkCanvas* canvas) { 1815 void GMMain::installFilter(SkCanvas* canvas) {
1698 if (FLAGS_forceBWtext) { 1816 if (FLAGS_forceBWtext) {
1699 canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref(); 1817 canvas->setDrawFilter(SkNEW(BWTextDrawFilter))->unref();
1700 } 1818 }
1701 } 1819 }
1702 1820
1703 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL) 1821 #if !defined(SK_BUILD_FOR_IOS) && !defined(SK_BUILD_FOR_NACL)
1704 int main(int argc, char * const argv[]) { 1822 int main(int argc, char * const argv[]) {
1705 return tool_main(argc, (char**) argv); 1823 return tool_main(argc, (char**) argv);
1706 } 1824 }
1707 #endif 1825 #endif
OLDNEW
« no previous file with comments | « gm/gm_error.h ('k') | gm/tests/outputs/compared-against-different-pixels-images/output-expected/stderr » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698