Implement multi-core builds in MSVS

visual_studio/NativeClientVSAddIn/NaCl.Build.CPPTasks/NaClCompile.cs

-
+
 using System;
 using System.Collections.Generic;
 using System.Text;
 using System.Collections;
 using System.IO;
 using System.Reflection;
 using System.Resources;
 using Microsoft.Build.Framework;
 using Microsoft.Build.Utilities;
+using System.Collections.Specialized;
 
 using System.Diagnostics;
 
 namespace NaCl.Build.CPPTasks
 {
     public class NaClCompile : ToolTask
     {
         private XamlParser m_XamlParser;
         private ITaskItem[] excludedInputPaths;
         private ITaskItem[] tlogReadFiles;
@@ skipping 11 matching lines @@
         [Required]
         public string PropertiesFile { get; set; }
 
         [Required]
         public ITaskItem[] Sources { get; set; }
 
         [Required]
         public string NaCLCompilerPath { get; set; }
 
         [Required]
-        public string OutputCommandLine { get; set; }
+        public bool OutputCommandLine { get; set; }
+
+        public int ProcessorNumber { get; set; }
 
         [Required]
         public string TrackerLogDirectory { get; set; }
-
+ 
+        protected override StringDictionary EnvironmentOverride
+        {
+            get { 
+                string show = OutputCommandLine ? "1" : "0";
+                string cores = Convert.ToString(ProcessorNumber);
+                return new StringDictionary() { 
+                      {"NACL_GCC_CORES", cores},
+                      {"NACL_GCC_SHOW_COMMANDS", show }
+                };
+            }
+        }
 
         protected override string GenerateFullPathToTool() { return ToolName; }
 
         public NaClCompile()
             : base(new ResourceManager("NaCl.Build.CPPTasks.Properties.Resources", Assembly.GetExecutingAssembly()))
         {
             this.pathToLog = string.Empty;
-            this.EnvironmentVariables = new string []{"CYGWIN=nodosfilewarning", "LC_CTYPE=C"};
+            this.EnvironmentVariables = new string[] { "CYGWIN=nodosfilewarning", "LC_CTYPE=C" };
         }
 
         protected IDictionary<string, string> GenerateCommandLinesFromTlog()
         {
             IDictionary<string, string> cmdLineDictionary = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
             string tlogFilename = this.TLogCommandFile.GetMetadata("FullPath");
             if (File.Exists(tlogFilename))
             {
                 using (StreamReader reader = File.OpenText(tlogFilename))
                 {
                     string filename = string.Empty;
                     for (string lineStr = reader.ReadLine(); lineStr != null; lineStr = reader.ReadLine())
                     {
                         if (lineStr.Length == 0 ||
                             (lineStr[0] == '^' && lineStr.Length == 1))
                         {
-                            Log.LogMessage(MessageImportance.High, "Invalid line in command tlog");
+                            Log.LogError("Invalid line in command tlog");
                             break;
                         }
                         else if (lineStr[0] == '^')
                         {
                             filename = lineStr.Substring(1);
                         }
                         else
                         {
                             cmdLineDictionary[filename] = lineStr;
                         }
                     }
                 }
             }
             return cmdLineDictionary;
         }
 
         protected override void LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance)
         {
             base.LogEventsFromTextOutput(GCCUtilities.Convert_Output_GCC_to_VS(singleLine), messageImportance);
         }
 
+        static string GetObjectFile(ITaskItem source)
+        {
+            string objectFilePath = Path.GetFullPath(source.GetMetadata("ObjectFileName"));
+            if ((File.GetAttributes(objectFilePath) & FileAttributes.Directory) != 0)

[binji, 2012/10/03 22:52:13]

might be nice to have a comment here explaining why this is necessary

[Sam Clegg, 2012/10/04 22:57:57]

Done.

+            {
+                objectFilePath = Path.Combine(objectFilePath, Path.GetFileName(source.ItemSpec));
+                objectFilePath = Path.ChangeExtension(objectFilePath, ".obj");
+            }
+            return objectFilePath;
+        }
+
         private void ConstructReadTLog(ITaskItem[] compiledSources, CanonicalTrackedOutputFiles outputs)
         {
             string trackerPath = Path.GetFullPath(TlogDirectory + ReadTLogFilenames[0]);
 
             //save tlog for sources not compiled during this execution
             TaskItem readTrackerItem = new TaskItem(trackerPath);
             CanonicalTrackedInputFiles files = new CanonicalTrackedInputFiles(new TaskItem[] { readTrackerItem }, Sources, outputs, false, false);
             files.RemoveEntriesForSource(compiledSources);
             files.SaveTlog();
 
             //add tlog information for compiled sources
             using (StreamWriter writer = new StreamWriter(trackerPath, true, Encoding.Unicode))
             {
                 foreach (ITaskItem source in compiledSources)
                 {
                     string sourcePath = Path.GetFullPath(source.ItemSpec).ToUpperInvariant();
-
-                    string objectFilePath = Path.GetFullPath(source.GetMetadata("ObjectFileName"));
+                    string objectFilePath = GetObjectFile(source);
                     string depFilePath = Path.ChangeExtension(objectFilePath, ".d");
 
                     try
                     {
                         if (File.Exists(depFilePath) == false)
                         {
                             Log.LogMessage(MessageImportance.High, depFilePath + " not found");
                         }
                         else
                         {
@@ skipping 81 matching lines @@
                 }
             }
         }
 
         protected string GenerateCommandLineFromProps(ITaskItem sourceFile)
         {
             StringBuilder commandLine = new StringBuilder(GCCUtilities.s_CommandLineLength);
 
             if (sourceFile != null)
             {
-                string sourcePath = GCCUtilities.Convert_Path_Windows_To_Posix(sourceFile.ToString());
-
                 // Remove rtti items as they are not relevant in C compilation and will produce warnings
                 if (SourceIsC(sourceFile.ToString()))
                 {
                     commandLine.Replace("-fno-rtti", "");
                     commandLine.Replace("-frtti", "");
                 }
 
                 //build command line from components and add required switches
                 string props = m_XamlParser.Parse(sourceFile);
                 commandLine.Append(props);
                 commandLine.Append(" -MD -c ");
-                commandLine.Append("\"" + sourcePath + "\"");
             }
 
             return commandLine.ToString();
         }
 
         protected ITaskItem[] MergeOutOfDateSources(ITaskItem[] outOfDateSourcesFromTracking, List<ITaskItem> outOfDateSourcesFromCommandLineChanges)
         {
             List<ITaskItem> mergedSources = new List<ITaskItem>(outOfDateSourcesFromTracking);
 
             foreach (ITaskItem item in outOfDateSourcesFromCommandLineChanges)
@@ skipping 31 matching lines @@
             else
             {
                 return false;
             }
         }
 
         private int Compile(string pathToTool)
         {
             int returnCode = 0;
 
-            foreach (ITaskItem sourceFileItem in CompileSourceList)
+            // Compute sources that can be compiled together.
+            Dictionary<string, List<ITaskItem>> srcGroups = new Dictionary<string, List<ITaskItem>>();

[binji, 2012/10/03 22:52:13]

nit: wrap line

[Sam Clegg, 2012/10/04 22:57:57]

Done.

+            foreach (ITaskItem sourceItem in CompileSourceList)
             {
+                string commandLine = GenerateCommandLineFromProps(sourceItem);
+                if (srcGroups.ContainsKey(commandLine))
+                {
+                    srcGroups[commandLine].Add(sourceItem);
+                }
+                else
+                {
+                    srcGroups.Add(commandLine, new List<ITaskItem> {sourceItem});
+                }
+            }
+
+            string pythonExe = "C:\\python_26_amd64\\files\\python.exe";

[binji, 2012/10/03 22:52:13]

Probably need a better way to find the python exe.

[Sam Clegg, 2012/10/04 22:57:57]

Yup.. will be sure to find a better way before I land this.

+            string pythonScript = Path.GetDirectoryName(Path.GetDirectoryName(PropertiesFile));
+            pythonScript = Path.Combine(pythonScript, "compiler_wrapper.py");
+
+            foreach (KeyValuePair<string, List<ITaskItem>> entry in srcGroups)
+            {
+                string commandLine = entry.Key;
+                string cmd = "\"" + pathToTool + "\" " + commandLine + "--";
+                List<ITaskItem> sources = entry.Value;
+
+                foreach (ITaskItem sourceItem in sources)
+                {
+                    string sourcePath = GCCUtilities.Convert_Path_Windows_To_Posix(sourceItem.ToString());

[binji, 2012/10/03 22:52:13]

nit: wrap line

[Sam Clegg, 2012/10/04 22:57:57]

Done.

+                    cmd += " \"" + sourcePath + "\"";
+                }
+
                 try
                 {
-                    string commandLine = GenerateCommandLineFromProps(sourceFileItem);
-
-                    base.Log.LogMessageFromText(Path.GetFileName(sourceFileItem.ToString()), MessageImportance.High);
-
-                    if (OutputCommandLine == "true")
-                    {
-                        string logMessage = pathToTool + " " + commandLine;
-                        Log.LogMessageFromText(logMessage, MessageImportance.High);
-                    }
-
-
-                    // compile
-                    returnCode = base.ExecuteTool(pathToTool, commandLine, string.Empty);
+                    // compile this group of sources
+                    returnCode = base.ExecuteTool(pythonExe, cmd, "\"" + pythonScript + "\"");
                 }
-                catch (Exception)
+                catch (Exception e)
                 {
+                    Log.LogMessage("compiler exception: {0}", e);
                     returnCode = base.ExitCode;
                 }
 
                 //abort if an error was encountered
                 if (returnCode != 0)
-                {
-                    return returnCode;
-                }
+                    break;
             }
+
+            Log.LogMessage(MessageImportance.Low, "compiler returned: {0}", returnCode);
             return returnCode;
         }
 
         protected override int ExecuteTool(string pathToTool, string responseFileCommands, string commandLineCommands)
         {
             if (File.Exists(pathToTool) == false)
             {
                 base.Log.LogMessageFromText("Unable to find NaCL compiler: " + pathToTool, MessageImportance.High);
                 return -1;
             }
@@ skipping 319 matching lines @@
 
         protected override Encoding ResponseFileEncoding
         {
             get
             {
                 return Encoding.ASCII;
             }
         }
     }
 }