Implement multi-core builds in MSVS

visual_studio/NativeClientVSAddIn/NaCl.Build.CPPTasks/NaClCompile.cs

-
+
 using System;
 using System.Collections.Generic;
 using System.Text;
 using System.Collections;
 using System.IO;
 using System.Reflection;
 using System.Resources;
+using System.Windows.Forms;
 using Microsoft.Build.Framework;
+using Microsoft.Win32;
 using Microsoft.Build.Utilities;
+using System.Collections.Specialized;
 
 using System.Diagnostics;
 
 namespace NaCl.Build.CPPTasks
 {
     public class NaClCompile : ToolTask
     {
         private XamlParser m_XamlParser;
         private ITaskItem[] excludedInputPaths;
         private ITaskItem[] tlogReadFiles;
@@ skipping 11 matching lines @@
         [Required]
         public string PropertiesFile { get; set; }
 
         [Required]
         public ITaskItem[] Sources { get; set; }
 
         [Required]
         public string NaCLCompilerPath { get; set; }
 
         [Required]
-        public string OutputCommandLine { get; set; }
+        public bool OutputCommandLine { get; set; }
+
+        public int ProcessorNumber { get; set; }
+
+        public bool MultiProcessorCompilation { get; set; }
 
         [Required]
         public string TrackerLogDirectory { get; set; }
 
+        protected override StringDictionary EnvironmentOverride
+        {
+            get {
+                string show = OutputCommandLine ? "1" : "0";
+                string cores = Convert.ToString(ProcessorNumber);
+                return new StringDictionary() {
+                      {"NACL_GCC_CORES", cores},
+                      {"NACL_GCC_SHOW_COMMANDS", show }
+                };
+            }
+        }
 
         protected override string GenerateFullPathToTool() { return ToolName; }
 
         public NaClCompile()
             : base(new ResourceManager("NaCl.Build.CPPTasks.Properties.Resources", Assembly.GetExecutingAssembly()))
         {
             this.pathToLog = string.Empty;
-            this.EnvironmentVariables = new string []{"CYGWIN=nodosfilewarning", "LC_CTYPE=C"};
+            this.EnvironmentVariables = new string[] { "CYGWIN=nodosfilewarning", "LC_CTYPE=C" };
         }
 
         protected IDictionary<string, string> GenerateCommandLinesFromTlog()
         {
             IDictionary<string, string> cmdLineDictionary = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
             string tlogFilename = this.TLogCommandFile.GetMetadata("FullPath");
             if (File.Exists(tlogFilename))
             {
                 using (StreamReader reader = File.OpenText(tlogFilename))
                 {
                     string filename = string.Empty;
                     for (string lineStr = reader.ReadLine(); lineStr != null; lineStr = reader.ReadLine())
                     {
                         if (lineStr.Length == 0 ||
                             (lineStr[0] == '^' && lineStr.Length == 1))
                         {
-                            Log.LogMessage(MessageImportance.High, "Invalid line in command tlog");
+                            Log.LogError("Invalid line in command tlog");
                             break;
                         }
                         else if (lineStr[0] == '^')
                         {
                             filename = lineStr.Substring(1);
                         }
                         else
                         {
                             cmdLineDictionary[filename] = lineStr;
                         }
                     }
                 }
             }
             return cmdLineDictionary;
         }
 
         protected override void LogEventsFromTextOutput(string singleLine, MessageImportance messageImportance)
         {
-            base.LogEventsFromTextOutput(GCCUtilities.Convert_Output_GCC_to_VS(singleLine), messageImportance);
+            base.LogEventsFromTextOutput(GCCUtilities.ConvertGCCOutput(singleLine), messageImportance);
+        }
+
+        static string GetObjectFile(ITaskItem source)
+        {
+            string objectFilePath = Path.GetFullPath(source.GetMetadata("ObjectFileName"));
+            // cl.exe will accept a folder name as the ObjectFileName in which case
+            // the objectfile is created as <ObjectFileName>/<basename>.obj.  Here
+            // we mimic this behaviour.
+            if ((File.GetAttributes(objectFilePath) & FileAttributes.Directory) != 0)
+            {
+                objectFilePath = Path.Combine(objectFilePath, Path.GetFileName(source.ItemSpec));
+                objectFilePath = Path.ChangeExtension(objectFilePath, ".obj");
+            }
+            return objectFilePath;
         }
 
         private void ConstructReadTLog(ITaskItem[] compiledSources, CanonicalTrackedOutputFiles outputs)
         {
             string trackerPath = Path.GetFullPath(TlogDirectory + ReadTLogFilenames[0]);
 
             //save tlog for sources not compiled during this execution
             TaskItem readTrackerItem = new TaskItem(trackerPath);
             CanonicalTrackedInputFiles files = new CanonicalTrackedInputFiles(new TaskItem[] { readTrackerItem }, Sources, outputs, false, false);
             files.RemoveEntriesForSource(compiledSources);
             files.SaveTlog();
 
             //add tlog information for compiled sources
             using (StreamWriter writer = new StreamWriter(trackerPath, true, Encoding.Unicode))
             {
                 foreach (ITaskItem source in compiledSources)
                 {
                     string sourcePath = Path.GetFullPath(source.ItemSpec).ToUpperInvariant();
-
-                    string objectFilePath = Path.GetFullPath(source.GetMetadata("ObjectFileName"));
+                    string objectFilePath = GetObjectFile(source);
                     string depFilePath = Path.ChangeExtension(objectFilePath, ".d");
 
                     try
                     {
                         if (File.Exists(depFilePath) == false)
                         {
                             Log.LogMessage(MessageImportance.High, depFilePath + " not found");
                         }
                         else
                         {
@@ skipping 39 matching lines @@
             string path = Path.Combine(TlogDirectory, WriteTLogFilename);
             TaskItem item = new TaskItem(path);
             CanonicalTrackedOutputFiles trackedFiles = new CanonicalTrackedOutputFiles(new TaskItem[] { item });
 
             foreach (ITaskItem sourceItem in compiledSources)
             {
                 //remove this entry associated with compiled source which is about to be recomputed
                 trackedFiles.RemoveEntriesForSource(sourceItem);
 
                 //add entry with updated information
-                trackedFiles.AddComputedOutputForSourceRoot( Path.GetFullPath(sourceItem.ItemSpec).ToUpperInvariant(),
-                                                             Path.GetFullPath(sourceItem.GetMetadata("ObjectFileName")).ToUpperInvariant());
+                trackedFiles.AddComputedOutputForSourceRoot(Path.GetFullPath(sourceItem.ItemSpec).ToUpperInvariant(),
+                                                            Path.GetFullPath(GetObjectFile(sourceItem)).ToUpperInvariant());
             }
 
             //output tlog
             trackedFiles.SaveTlog();
 
             return trackedFiles;
         }
 
         private void OutputCommandTrackerLog(ITaskItem[] compiledSources)
         {
@@ skipping 14 matching lines @@
             {
                 foreach (KeyValuePair<string, string> p in commandLines)
                 {
                     string keyLine = "^" + p.Key;
                     writer.WriteLine(keyLine);
                     writer.WriteLine(p.Value);
                 }
             }
         }
 
-        protected string GenerateCommandLineFromProps(ITaskItem sourceFile)
+        protected string GenerateCommandLineFromProps(ITaskItem sourceFile, bool fullOutputName=false)
         {
             StringBuilder commandLine = new StringBuilder(GCCUtilities.s_CommandLineLength);
 
             if (sourceFile != null)
             {
-                string sourcePath = GCCUtilities.Convert_Path_Windows_To_Posix(sourceFile.ToString());
-
                 // Remove rtti items as they are not relevant in C compilation and will produce warnings
                 if (SourceIsC(sourceFile.ToString()))
                 {
                     commandLine.Replace("-fno-rtti", "");
                     commandLine.Replace("-frtti", "");
                 }
 
                 //build command line from components and add required switches
-                string props = m_XamlParser.Parse(sourceFile);
+                string props = m_XamlParser.Parse(sourceFile, fullOutputName);
                 commandLine.Append(props);
                 commandLine.Append(" -MD -c ");
-                commandLine.Append("\"" + sourcePath + "\"");
             }
 
             return commandLine.ToString();
         }
 
         protected ITaskItem[] MergeOutOfDateSources(ITaskItem[] outOfDateSourcesFromTracking, List<ITaskItem> outOfDateSourcesFromCommandLineChanges)
         {
             List<ITaskItem> mergedSources = new List<ITaskItem>(outOfDateSourcesFromTracking);
 
             foreach (ITaskItem item in outOfDateSourcesFromCommandLineChanges)
@@ skipping 29 matching lines @@
                 return true;
             }
             else
             {
                 return false;
             }
         }
 
         private int Compile(string pathToTool)
         {
+            // If multiprocess complication is enabled (not the VS default)
+            // and the number of processors to use is not 1, then use the
+            // compiler_wrapper python script to run multiple instances of
+            // gcc
+            if (MultiProcessorCompilation && ProcessorNumber != 1)
+            {
+
+                string envvar = (string)Registry.GetValue("HKEY_CURRENT_USER\\Environment", "PATH", "");
+                List<string> pathList = new List<string>(envvar.Split(';'));
+                envvar = (string)Registry.GetValue("HKEY_LOCAL_MACHINE\\System\\CurrentControlSet\\Control\\Session Manager\\Environment", "PATH", "");
+                pathList.AddRange(new List<string>(envvar.Split(';')));
+                string pythonExe = null;
+                foreach (string path in pathList)
+                {
+                    string testPath = Path.Combine(path, "python.bat");
+                    if (File.Exists(testPath))
+                    {
+                        pythonExe = testPath;
+                        break;
+                    }
+                    testPath = Path.Combine(path, "python.exe");
+                    if (File.Exists(testPath))
+                    {
+                        pythonExe = testPath;
+                        break;
+                    }
+                }
+
+                if (pythonExe == null)
+                {
+                    MessageBox.Show("Multi-processor Compilation with NaCl requires that python available in the PATH.\n" +
+                                    "Please disable Multi-processor Compilation in the project properties or add python " +
+                                    "to the your PATH\n" +
+                                    "Falling back to serial compilation.\n");
+                }
+                else
+                {
+                    return CompileParallel(pathToTool, pythonExe);
+                }
+            }
+            return CompileSerial(pathToTool);
+        }
+
+        private int CompileSerial(string pathToTool)
+        {
             int returnCode = 0;
 
-            foreach (ITaskItem sourceFileItem in CompileSourceList)
+            foreach (ITaskItem sourceItem in CompileSourceList)
             {
                 try
                 {
-                    string commandLine = GenerateCommandLineFromProps(sourceFileItem);
+                    string commandLine = GenerateCommandLineFromProps(sourceItem, true);
+                    commandLine += "\"" + GCCUtilities.ConvertPathWindowsToPosix(sourceItem.ToString()) + "\"";
 
-                    base.Log.LogMessageFromText(Path.GetFileName(sourceFileItem.ToString()), MessageImportance.High);
-
-                    if (OutputCommandLine == "true")
+                    if (OutputCommandLine)
                     {
                         string logMessage = pathToTool + " " + commandLine;
-                        Log.LogMessageFromText(logMessage, MessageImportance.High);
+                        Log.LogMessage(logMessage);
+                    }
+                    else
+                    {
+                        base.Log.LogMessage(Path.GetFileName(sourceItem.ToString()));
                     }
 
 
                     // compile
                     returnCode = base.ExecuteTool(pathToTool, commandLine, string.Empty);
                 }
                 catch (Exception)
                 {
                     returnCode = base.ExitCode;
                 }
 
                 //abort if an error was encountered
                 if (returnCode != 0)
                 {
                     return returnCode;
                 }
             }
             return returnCode;
         }
 
+        private int CompileParallel(string pathToTool, string pythonExe)
+        {
+            int returnCode = 0;
+
+            // Compute sources that can be compiled together.
+            Dictionary<string, List<ITaskItem>> srcGroups =
+                    new Dictionary<string, List<ITaskItem>>();
+
+            foreach (ITaskItem sourceItem in CompileSourceList)
+            {
+                string commandLine = GenerateCommandLineFromProps(sourceItem);
+                if (srcGroups.ContainsKey(commandLine))
+                {
+                    srcGroups[commandLine].Add(sourceItem);
+                }
+                else
+                {
+                    srcGroups.Add(commandLine, new List<ITaskItem> {sourceItem});
+                }
+            }
+
+            string pythonScript = Path.GetDirectoryName(Path.GetDirectoryName(PropertiesFile));
+            pythonScript = Path.Combine(pythonScript, "compiler_wrapper.py");
+
+            foreach (KeyValuePair<string, List<ITaskItem>> entry in srcGroups)
+            {
+                string commandLine = entry.Key;
+                string cmd = "\"" + pathToTool + "\" " + commandLine + "--";
+                List<ITaskItem> sources = entry.Value;
+
+                foreach (ITaskItem sourceItem in sources)
+                {
+                    string src = GCCUtilities.ConvertPathWindowsToPosix(sourceItem.ToString());
+                    cmd += " \"" + src + "\"";
+                }
+
+                try
+                {
+                    // compile this group of sources
+                    returnCode = base.ExecuteTool(pythonExe, cmd, "\"" + pythonScript + "\"");
+                }
+                catch (Exception e)
+                {
+                    Log.LogMessage("compiler exception: {0}", e);
+                    returnCode = base.ExitCode;
+                }
+
+                //abort if an error was encountered
+                if (returnCode != 0)
+                    break;
+            }
+
+            Log.LogMessage(MessageImportance.Low, "compiler returned: {0}", returnCode);
+            return returnCode;
+        }
+
         protected override int ExecuteTool(string pathToTool, string responseFileCommands, string commandLineCommands)
         {
             if (File.Exists(pathToTool) == false)
             {
                 base.Log.LogMessageFromText("Unable to find NaCL compiler: " + pathToTool, MessageImportance.High);
                 return -1;
             }
 
             int returnCode = -1;
 
@@ skipping 17 matching lines @@
         {
             if (this.TrackFileAccess || this.MinimalRebuildFromTracking)
             {
                 this.SetTrackerLogPaths();
             }
 
             //check if full recompile is required otherwise perform incremental
             if (this.ForcedRebuildRequired() || this.MinimalRebuildFromTracking == false)
             {
                 this.CompileSourceList = this.Sources;
-                if ((this.CompileSourceList == null) || (this.CompileSourceList.Length == 0))
+                if (this.CompileSourceList == null || this.CompileSourceList.Length == 0)
                 {
                     this.SkippedExecution = true;
                 }
             }
             else
             {
                 //retrieve list of sources out of date due to command line changes
                 List<ITaskItem> outOfDateSourcesFromCommandLineChanges = this.GetOutOfDateSourcesFromCommandLineChanges();
 
                 //retrieve sources out of date due to tracking
@@ skipping 278 matching lines @@
 
         protected override Encoding ResponseFileEncoding
         {
             get
             {
                 return Encoding.ASCII;
             }
         }
     }
 }