Make gcmole execute in parallel.

tools/gcmole/gcmole.lua

Index: tools/gcmole/gcmole.lua
diff --git a/tools/gcmole/gcmole.lua b/tools/gcmole/gcmole.lua
index d287f7b9122bf3d32e7bf02c04f7ccd8761e4be5..0d7675a9fac01776f997278a7b4b0688ece8759b 100644
--- a/tools/gcmole/gcmole.lua
+++ b/tools/gcmole/gcmole.lua
@@ -34,6 +34,9 @@ local FLAGS = {
    -- Do not build gcsuspects file and reuse previously generated one.
    reuse_gcsuspects = false;
 
+   -- Don't use parallel python runner.
+   sequential = false;
+
    -- Print commands to console before executing them.
    verbose = false;
 
@@ -111,18 +114,75 @@ local function MakeClangCommandLine(plugin, plugin_args, triple, arch_define)
       .. " -Ithird_party/icu/source/i18n"
 end
 
+function IterTable(t)

[Vyacheslav Egorov (Google), 2015/02/18 10:56:49]

I'd write it 

local function IterTable(t)
  return coroutine.wrap(function ()
    for i, v in ipairs(t) do
      coroutine.yield(v)
    end
  end)
end

or (which arguably is uglier)

local function IterTable(t)
  local f, n = ipairs(t), 0
  return function ()
    local v
    n, v = ipairs(t, n)
    return n
  end
end

but see the suggestion below which can eliminate the need for it entirely.

[Vyacheslav Egorov (Google), 2015/02/18 10:59:51]

my comment should obviously read:
at the end

[Michael Achenbach, 2015/02/18 14:48:59]

Done.

+   -- Iterator for arrays. Compatible with lua 5.0 and 5.1 as it doesn't
+   -- use getn.
+   local n = 0
+   for _ in pairs(t) do n = n + 1 end
+
+   local i = 0
+   return function ()
+      i = i + 1
+      if i <= n then return t[i] end
+   end
+end
+
+local function SplitResults(lines, func)
+   -- Splits the output of parallel.py and calls func on each result.
+   -- Bails out in case of an error in one of the executions.
+   local current = {}
+   local filename = ""
+   for line in lines do
+      local new_file = line:match "^______________ (.*)$"
+      local code = line:match "^______________ finish (%d+) ______________$"
+      if code then
+         if tonumber(code) > 0 then
+            log(table.concat(current, "\n"))
+            log("Failed to examine " .. filename)
+            return false
+         end
+         log("-- %s", filename)
+         func(filename, IterTable(current))
+      elseif new_file then
+         filename = new_file
+         current = {}
+      else
+         table.insert(current, line)
+      end
+   end
+   return true
+end
+
 function InvokeClangPluginForEachFile(filenames, cfg, func)
    local cmd_line = MakeClangCommandLine(cfg.plugin,
                                          cfg.plugin_args,
                                          cfg.triple,
                                          cfg.arch_define)
-   for _, filename in ipairs(filenames) do
-      log("-- %s", filename)
-      local action = cmd_line .. " " .. filename .. " 2>&1"
+   if FLAGS.sequential then
+      log("** Sequential execution.")

[Michael Achenbach, 2015/02/18 10:04:54]

This is the original piece of code c/p 1:1 (codereview not smart)

+      for _, filename in ipairs(filenames) do
+         log("-- %s", filename)
+         local action = cmd_line .. " " .. filename .. " 2>&1"
+         if FLAGS.verbose then print('popen ', action) end
+         local pipe = io.popen(action)
+         func(filename, pipe:lines())
+         local success = pipe:close()
+         if not success then error("Failed to run: " .. action) end
+      end
+   else
+      log("** Parallel execution.")
+      local parallel_cmd_line = {
+         "python",
+         "tools/gcmole/parallel.py",
+         "\"" .. cmd_line .. "\""
+      }
+      for _, filename in ipairs(filenames) do
+         table.insert(parallel_cmd_line, filename)
+      end
+      action = table.concat(parallel_cmd_line, " ")

[Vyacheslav Egorov (Google), 2015/02/18 10:56:49]

arguably you can just do

action = table.concat(parallel_cmd_line, " ") .. " " .. table.concat(filenames,
" ")

and kill the loop above

[Michael Achenbach, 2015/02/18 14:48:59]

Made it even shorter.

       if FLAGS.verbose then print('popen ', action) end
       local pipe = io.popen(action)
-      func(filename, pipe:lines())
-      local success = pipe:close()
+      local success = SplitResults(pipe:lines(), func) and pipe:close()

[Vyacheslav Egorov (Google), 2015/02/18 10:56:49]

pipe will not be closed if SplitResults returns false. 

[Michael Achenbach, 2015/02/18 14:48:59]

Done.

       if not success then error("Failed to run: " .. action) end
    end
 end