trim DM fat

dm/DM.cpp

 /*
  * Copyright 2013 Google Inc.
  *
  * Use of this source code is governed by a BSD-style license that can be
  * found in the LICENSE file.
  */
 
 #include "CrashHandler.h"
 #include "DMJsonWriter.h"
 #include "DMSrcSink.h"
@@ skipping 53 matching lines @@
         "this list, no image is written for that result.");
 
 DEFINE_int32(shards, 1, "We're splitting source data into this many shards.");
 DEFINE_int32(shard,  0, "Which shard do I run?");
 
 __SK_FORCE_IMAGE_DECODER_LINKING;
 using namespace DM;
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
-static double now_ms() { return SkTime::GetNSecs() * 1e-6; }
-
 SK_DECLARE_STATIC_MUTEX(gFailuresMutex);
 static SkTArray<SkString> gFailures;
 
-static void fail(ImplicitString err) {
+static void fail(const SkString& err) {
     SkAutoMutexAcquire lock(gFailuresMutex);
     SkDebugf("\n\nFAILURE: %s\n\n", err.c_str());
     gFailures.push_back(err);
 }
 
 
 // We use a spinlock to make locking this in a signal handler _somewhat_ safe.
 SK_DECLARE_STATIC_SPINLOCK(gMutex);
-static int32_t                   gPending;
-static SkTArray<SkString>        gRunning;
-static SkTHashMap<SkString, int> gNoteTally;
+static int32_t            gPending;
+static SkTArray<SkString> gRunning;
 
-static void done(double ms,
-                 ImplicitString config, ImplicitString src, ImplicitString srcOptions,
-                 ImplicitString name, ImplicitString note, ImplicitString log) {
-    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
-                                                srcOptions.c_str(), name.c_str());
+static void done(const char* config, const char* src, const char* srcOptions, const char* name) {
+    SkString id = SkStringPrintf("%s %s %s %s", config, src, srcOptions, name);
     int pending;
     {
         SkAutoTAcquire<SkPODSpinlock> lock(gMutex);
         for (int i = 0; i < gRunning.count(); i++) {
             if (gRunning[i] == id) {
                 gRunning.removeShuffle(i);
                 break;
             }
         }
-        if (!note.isEmpty()) {
-            if (int* tally = gNoteTally.find(note)) {
-                *tally += 1;
-            } else {
-                gNoteTally.set(note, 1);
-            }
-        }
         pending = --gPending;
     }
     // We write our dm.json file every once in a while in case we crash.
     // Notice this also handles the final dm.json when pending == 0.
     if (pending % 500 == 0) {
         JsonWriter::DumpJson();
     }
 }
 
-static void start(ImplicitString config, ImplicitString src,
-                  ImplicitString srcOptions, ImplicitString name) {
-    SkString id = SkStringPrintf("%s %s %s %s", config.c_str(), src.c_str(),
-                                                srcOptions.c_str(), name.c_str());
+static void start(const char* config, const char* src, const char* srcOptions, const char* name) {
+    SkString id = SkStringPrintf("%s %s %s %s", config, src, srcOptions, name);
     SkAutoTAcquire<SkPODSpinlock> lock(gMutex);
     gRunning.push_back(id);
 }
 
 static void print_status() {
     static SkMSec start_ms = SkTime::GetMSecs();
 
     int curr = sk_tools::getCurrResidentSetSizeMB(),
         peak = sk_tools::getMaxResidentSetSizeMB();
     SkString elapsed = HumanizeMs(SkTime::GetMSecs() - start_ms);
@@ skipping 22 matching lines @@
                 print_status();
             });
         }
     }
 #endif
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
 struct Gold : public SkString {
     Gold() : SkString("") {}
-    Gold(ImplicitString sink, ImplicitString src, ImplicitString srcOptions,
-         ImplicitString name, ImplicitString md5)
+    Gold(const SkString& sink, const SkString& src,
+         const SkString& srcOptions, const SkString& name,
+         const SkString& md5)
         : SkString("") {
         this->append(sink);
         this->append(src);
         this->append(srcOptions);
         this->append(name);
         this->append(md5);
     }
     struct Hash {
         uint32_t operator()(const Gold& g) const {
             return SkGoodHash()((const SkString&)g);
@@ skipping 34 matching lines @@
             gUninterestingHashes.add(hash);
         }
         SkDebugf("FYI: loaded %d distinct uninteresting hashes from %d lines\n",
                  gUninterestingHashes.count(), hashes.count());
     }
 }
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
 struct TaggedSrc : public SkAutoTDelete<Src> {
-    ImplicitString tag;
-    ImplicitString options;
+    SkString tag;
+    SkString options;
 };
 
 struct TaggedSink : public SkAutoTDelete<Sink> {
     SkString tag;
 };
 
 static const bool kMemcpyOK = true;
 
 static SkTArray<TaggedSrc,  kMemcpyOK>  gSrcs;
 static SkTArray<TaggedSink, kMemcpyOK> gSinks;
 
 static bool in_shard() {
     static int N = 0;
     return N++ % FLAGS_shards == FLAGS_shard;
 }
 
-static void push_src(ImplicitString tag, ImplicitString options, Src* s) {
+static void push_src(const char* tag, ImplicitString options, Src* s) {
     SkAutoTDelete<Src> src(s);
     if (in_shard() &&
-        FLAGS_src.contains(tag.c_str()) &&
+        FLAGS_src.contains(tag) &&
         !SkCommandLineFlags::ShouldSkip(FLAGS_match, src->name().c_str())) {
         TaggedSrc& s = gSrcs.push_back();
         s.reset(src.detach());
         s.tag = tag;
         s.options = options;
     }
 }
 
 static void push_codec_src(Path path, CodecSrc::Mode mode, CodecSrc::DstColorType dstColorType,
         SkAlphaType dstAlphaType, float scale) {
@@ skipping 593 matching lines @@
 
     png_destroy_write_struct(&png, &info);
     fclose(f);
     return true;
 }
 
 static bool match(const char* needle, const char* haystack) {
     return 0 == strcmp("_", needle) || nullptr != strstr(haystack, needle);
 }
 
-static ImplicitString is_blacklisted(const char* sink, const char* src,
-                                     const char* srcOptions, const char* name) {
+static bool is_blacklisted(const char* sink, const char* src,
+                           const char* srcOptions, const char* name) {
     for (int i = 0; i < FLAGS_blacklist.count() - 3; i += 4) {
         if (match(FLAGS_blacklist[i+0], sink) &&
             match(FLAGS_blacklist[i+1], src) &&
             match(FLAGS_blacklist[i+2], srcOptions) &&
             match(FLAGS_blacklist[i+3], name)) {
-            return SkStringPrintf("%s %s %s %s",
-                                  FLAGS_blacklist[i+0], FLAGS_blacklist[i+1],
-                                  FLAGS_blacklist[i+2], FLAGS_blacklist[i+3]);
+            return true;
         }
     }
-    return "";
+    return false;
 }
 
 // Even when a Task Sink reports to be non-threadsafe (e.g. GPU), we know things like
 // .png encoding are definitely thread safe.  This lets us offload that work to CPU threads.
 static SkTaskGroup gDefinitelyThreadSafeWork;
 
 // The finest-grained unit of work we can run: draw a single Src into a single Sink,
 // report any errors, and perhaps write out the output: a .png of the bitmap, or a raw stream.
 struct Task {
     Task(const TaggedSrc& src, const TaggedSink& sink) : src(src), sink(sink) {}
     const TaggedSrc&  src;
     const TaggedSink& sink;
 
     static void Run(const Task& task) {
         SkString name = task.src->name();
 
-        // We'll skip drawing this Src/Sink pair if:
-        //   - the Src vetoes the Sink;
-        //   - this Src / Sink combination is on the blacklist;
-        //   - it's a dry run.
-        SkString note(task.src->veto(task.sink->flags()) ? " (veto)" : "");
-        SkString whyBlacklisted = is_blacklisted(task.sink.tag.c_str(), task.src.tag.c_str(),
-                                                 task.src.options.c_str(), name.c_str());
-        if (!whyBlacklisted.isEmpty()) {
-            note.appendf(" (--blacklist %s)", whyBlacklisted.c_str());
-        }
-
         SkString log;
-        auto timerStart = now_ms();
-        if (!FLAGS_dryRun && note.isEmpty()) {
+        if (!FLAGS_dryRun) {
             SkBitmap bitmap;
             SkDynamicMemoryWStream stream;
-            start(task.sink.tag.c_str(), task.src.tag, task.src.options, name.c_str());
+            start(task.sink.tag.c_str(), task.src.tag.c_str(),
+                  task.src.options.c_str(), name.c_str());
             Error err = task.sink->draw(*task.src, &bitmap, &stream, &log);
             if (!err.isEmpty()) {
                 if (err.isFatal()) {
                     fail(SkStringPrintf("%s %s %s %s: %s",
                                         task.sink.tag.c_str(),
                                         task.src.tag.c_str(),
                                         task.src.options.c_str(),
                                         name.c_str(),
                                         err.c_str()));
                 } else {
-                    note.appendf(" (skipped: %s)", err.c_str());
-                    auto elapsed = now_ms() - timerStart;
-                    done(elapsed, task.sink.tag.c_str(), task.src.tag, task.src.options,
-                         name, note, log);
+                    done(task.sink.tag.c_str(), task.src.tag.c_str(),
+                         task.src.options.c_str(), name.c_str());
                     return;
                 }
             }
 
             // We're likely switching threads here, so we must capture by value, [=] or [foo,bar].
             SkStreamAsset* data = stream.detachAsStream();
             gDefinitelyThreadSafeWork.add([task,name,bitmap,data]{
                 SkAutoTDelete<SkStreamAsset> ownedData(data);
 
                 // Why doesn't the copy constructor do this when we have pre-locked pixels?
@@ skipping 19 matching lines @@
                         }
                     }
                     SkMD5::Digest digest;
                     hash.finish(digest);
                     for (int i = 0; i < 16; i++) {
                         md5.appendf("%02x", digest.data[i]);
                     }
                 }
 
                 if (!FLAGS_readPath.isEmpty() &&
-                    !gGold.contains(Gold(task.sink.tag.c_str(), task.src.tag.c_str(),
-                                         task.src.options.c_str(), name, md5))) {
+                    !gGold.contains(Gold(task.sink.tag, task.src.tag,
+                                         task.src.options, name, md5))) {
                     fail(SkStringPrintf("%s not found for %s %s %s %s in %s",
                                         md5.c_str(),
                                         task.sink.tag.c_str(),
                                         task.src.tag.c_str(),
                                         task.src.options.c_str(),
                                         name.c_str(),
                                         FLAGS_readPath[0]));
                 }
 
                 if (!FLAGS_writePath.isEmpty()) {
                     const char* ext = task.sink->fileExtension();
                     if (data->getLength()) {
                         WriteToDisk(task, md5, ext, data, data->getLength(), nullptr);
                         SkASSERT(bitmap.drawsNothing());
                     } else if (!bitmap.drawsNothing()) {
                         WriteToDisk(task, md5, ext, nullptr, 0, &bitmap);
                     }
                 }
             });
         }
-        auto elapsed = now_ms() - timerStart;
-        done(elapsed, task.sink.tag.c_str(), task.src.tag.c_str(), task.src.options.c_str(),
-             name, note, log);
+        done(task.sink.tag.c_str(), task.src.tag.c_str(), task.src.options.c_str(), name.c_str());
     }
 
     static void WriteToDisk(const Task& task,
                             SkString md5,
                             const char* ext,
                             SkStream* data, size_t len,
                             const SkBitmap* bitmap) {
         JsonWriter::BitmapResult result;
         result.name          = task.src->name();
-        result.config        = task.sink.tag.c_str();
+        result.config        = task.sink.tag;
         result.sourceType    = task.src.tag;
         result.sourceOptions = task.src.options;
         result.ext           = ext;
         result.md5           = md5;
         JsonWriter::AddBitmapResult(result);
 
         // If an MD5 is uninteresting, we want it noted in the JSON file,
         // but don't want to dump it out as a .png (or whatever ext is).
         if (gUninterestingHashes.contains(md5)) {
             return;
@@ skipping 79 matching lines @@
         void reportFailed(const skiatest::Failure& failure) override {
             fail(failure.toString());
             JsonWriter::AddTestFailure(failure);
         }
         bool allowExtendedTest() const override {
             return FLAGS_pathOpsExtended;
         }
         bool verbose() const override { return FLAGS_veryVerbose; }
     } reporter;
 
-    SkString note;
-    SkString whyBlacklisted = is_blacklisted("_", "tests", "_", test.name);
-    if (!whyBlacklisted.isEmpty()) {
-        note.appendf(" (--blacklist %s)", whyBlacklisted.c_str());
-    }
-
-    auto timerStart = now_ms();
-    if (!FLAGS_dryRun && whyBlacklisted.isEmpty()) {
+    if (!FLAGS_dryRun && !is_blacklisted("_", "tests", "_", test.name)) {
         start("unit", "test", "", test.name);
         GrContextFactory factory;
         test.proc(&reporter, &factory);
     }
-    done(now_ms()-timerStart, "unit", "test", "", test.name, note, "");
+    done("unit", "test", "", test.name);
 }
 
 /*~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~*/
 
 DEFINE_int32(status_sec, 15, "Print status this often (and if we crash).");
 
 SkThread* start_status_thread() {
     auto thread = new SkThread([] (void*) {
         for (;;) {
             print_status();
@@ skipping 51 matching lines @@
     SkDebugf("%d srcs * %d sinks + %d tests == %d tasks",
              gSrcs.count(), gSinks.count(), gParallelTests.count() + gSerialTests.count(), gPending);
     SkAutoTDelete<SkThread> statusThread(start_status_thread());
 
     // Kick off as much parallel work as we can, making note of any serial work we'll need to do.
     SkTaskGroup parallel;
     SkTArray<Task> serial;
 
     for (auto& sink : gSinks)
     for (auto&  src : gSrcs) {
+        if (src->veto(sink->flags()) ||
+            is_blacklisted(sink.tag.c_str(), src.tag.c_str(),
+                           src.options.c_str(), src->name().c_str())) {
+            SkAutoTAcquire<SkPODSpinlock> lock(gMutex);
+            gPending--;
+            continue;
+        }
+
         Task task(src, sink);
         if (src->serial() || sink->serial()) {
             serial.push_back(task);
         } else {
             parallel.add([task] { Task::Run(task); });
         }
     }
     for (auto test : gParallelTests) {
         parallel.add([test] { run_test(test); });
     }
 
     // With the parallel work running, run serial tasks and tests here on main thread.
     for (auto task : serial) { Task::Run(task); }
     for (auto test : gSerialTests) { run_test(test); }
 
     // Wait for any remaining parallel work to complete (including any spun off of serial tasks).
     parallel.wait();
     gDefinitelyThreadSafeWork.wait();
 
     // We'd better have run everything.
     SkASSERT(gPending == 0);
 
     // At this point we're back in single-threaded land.
     sk_tool_utils::release_portable_typefaces();
 
-    if (FLAGS_verbose && gNoteTally.count() > 0) {
-        SkDebugf("\nNote tally:\n");
-        gNoteTally.foreach([](const SkString& note, int* tally) {
-            SkDebugf("%dx\t%s\n", *tally, note.c_str());
-        });
-    }
-
-    SkDebugf("\n");
     if (gFailures.count() > 0) {
         SkDebugf("Failures:\n");
         for (int i = 0; i < gFailures.count(); i++) {
             SkDebugf("\t%s\n", gFailures[i].c_str());
         }
         SkDebugf("%d failures\n", gFailures.count());
         return 1;
     }
 
 #ifdef SK_PDF_IMAGE_STATS
@@ skipping 92 matching lines @@
                                                            Reporter* reporter,
                                                            GrContextFactory* factory);
 } // namespace skiatest
 
 #if !defined(SK_BUILD_FOR_IOS)
 int main(int argc, char** argv) {
     SkCommandLineFlags::Parse(argc, argv);
     return dm_main();
 }
 #endif