OLD | NEW |
| (Empty) |
1 var fs = require('fs'); | |
2 var util = require('util'); | |
3 var exec = require('child_process').exec; | |
4 var path = require('path'); | |
5 | |
6 // We have numProcesses extraction tasks running simultaneously to improve | |
7 // performance. If your machine is slow, you may need to dial back the | |
8 // parallelism. | |
9 var numProcesses = 8; | |
10 | |
11 var db = {}; | |
12 var metadata = {}; | |
13 var USE_VM = false; | |
14 | |
15 // Warning: START_DART_MESSAGE must match the value hardcoded in extract.dart | |
16 // TODO(jacobr): figure out a cleaner way to parse this data. | |
17 var START_DART_MESSAGE = "START_DART_MESSAGE_UNIQUE_IDENTIFIER"; | |
18 var END_DART_MESSAGE = "END_DART_MESSAGE_UNIQUE_IDENTIFIER"; | |
19 | |
20 var domTypes = JSON.parse(fs.readFileSync('data/domTypes.json', | |
21 'utf8').toString()); | |
22 var cacheData = JSON.parse(fs.readFileSync('output/crawl/cache.json', | |
23 'utf8').toString()); | |
24 var dartIdl = JSON.parse(fs.readFileSync('data/dartIdl.json', | |
25 'utf8').toString()); | |
26 | |
27 try { | |
28 fs.mkdirSync('output/extract'); | |
29 } catch (e) { | |
30 // It doesn't matter if the directories already exist. | |
31 } | |
32 | |
33 var errorFiles = []; | |
34 // TODO(jacobr): blacklist these types as we can't get good docs for them. | |
35 // ["Performance"] | |
36 | |
37 function parseFile(type, onDone, entry, file, searchResultIndex) { | |
38 var inputFile; | |
39 try { | |
40 inputFile = fs.readFileSync("output/crawl/" + file, 'utf8').toString(); | |
41 } catch (e) { | |
42 console.warn("Couldn't read: " + file); | |
43 onDone(); | |
44 return; | |
45 } | |
46 | |
47 var inputFileRaw = inputFile; | |
48 // Cached pages have multiple DOCTYPE tags. Strip off the first one so that | |
49 // we have valid HTML. | |
50 // TODO(jacobr): use a regular expression instead of indexOf. | |
51 if (inputFile.toLowerCase().indexOf("<!doctype") == 0) { | |
52 var matchIndex = inputFile.toLowerCase().indexOf("<!doctype", 1); | |
53 if (matchIndex == -1) { | |
54 // not a cached page. | |
55 inputFile = inputFileRaw; | |
56 } else { | |
57 inputFile = inputFile.substr(matchIndex); | |
58 } | |
59 } | |
60 | |
61 // Disable all existing javascript in the input file to speed up parsing and | |
62 // avoid conflicts between our JS and the JS in the file. | |
63 inputFile = inputFile.replace(/<script type="text\/javascript"/g, | |
64 '<script type="text/ignored"'); | |
65 | |
66 var endBodyIndex = inputFile.lastIndexOf("</body>"); | |
67 if (endBodyIndex == -1) { | |
68 // Some files are missing a closing body tag. | |
69 endBodyIndex = inputFile.lastIndexOf("</html>"); | |
70 } | |
71 if (endBodyIndex == -1) { | |
72 if (inputFile.indexOf("Error 404 (Not Found)") != -1) { | |
73 console.warn("Skipping 404 file: " + file); | |
74 onDone(); | |
75 return; | |
76 } | |
77 throw "Unexpected file format for " + file; | |
78 } | |
79 | |
80 inputFile = inputFile.substring(0, endBodyIndex) + | |
81 '<script type="text/javascript">\n' + | |
82 ' if (window.layoutTestController) {\n' + | |
83 ' var controller = window.layoutTestController;\n' + | |
84 ' controller.dumpAsText();\n' + | |
85 ' controller.waitUntilDone();\n' + | |
86 ' }\n' + | |
87 'window.addEventListener("message", receiveMessage, false);\n' + | |
88 'function receiveMessage(event) {\n' + | |
89 ' if (event.data.indexOf("' + START_DART_MESSAGE + '") != 0) return;\n' + | |
90 ' console.log(event.data + "' + END_DART_MESSAGE + '");\n' + | |
91 // We feature detect whether the browser supports layoutTestController | |
92 // so we only clear the document content when running in the test shell | |
93 // and not when debugging using a normal browser. | |
94 ' if (window.layoutTestController) {\n' + | |
95 ' document.documentElement.textContent = "";\n' + | |
96 ' window.layoutTestController.notifyDone();\n' + | |
97 ' }\n' + | |
98 '}\n' + | |
99 '</script>\n' + | |
100 (USE_VM ? | |
101 '<script type="application/dart" src="../../extract.dart"></script>' : | |
102 '<script type="text/javascript" src="../../output/extract.dart.js">' + | |
103 '</script>') + | |
104 '\n' + inputFile.substring(endBodyIndex); | |
105 | |
106 console.log("Processing: " + file); | |
107 var absoluteDumpFileName = path.resolve("output/extract/" + file); | |
108 fs.writeFileSync(absoluteDumpFileName, inputFile, 'utf8'); | |
109 var parseArgs = { | |
110 type: type, | |
111 searchResult: entry, | |
112 dartIdl: dartIdl[type] | |
113 }; | |
114 fs.writeFileSync(absoluteDumpFileName + ".json", JSON.stringify(parseArgs), | |
115 'utf8'); | |
116 | |
117 /* | |
118 // TODO(jacobr): Make this run on platforms other than OS X. | |
119 var cmd = '../../../client/tests/drt/Content Shell.app/Contents/MacOS/' + | |
120 Content Shell' + absoluteDumpFileName; | |
121 */ | |
122 // TODO(eub): Make this run on platforms other than Linux. | |
123 var cmd = '../../../client/tests/drt/content_shell ' + absoluteDumpFileName; | |
124 console.log(cmd); | |
125 exec(cmd, | |
126 function (error, stdout, stderr) { | |
127 var msgIndex = stdout.indexOf(START_DART_MESSAGE); | |
128 console.log('all: ' + stdout); | |
129 console.log('stderr: ' + stderr); | |
130 if (error !== null) { | |
131 console.log('exec error: ' + error); | |
132 } | |
133 | |
134 // TODO(jacobr): use a regexp. | |
135 var msg = stdout.substring(msgIndex + START_DART_MESSAGE.length); | |
136 msg = msg.substring(0, msg.indexOf(END_DART_MESSAGE)); | |
137 if (!(type in db)) { | |
138 db[type] = []; | |
139 } | |
140 try { | |
141 db[type][searchResultIndex] = JSON.parse(msg); | |
142 } catch(e) { | |
143 // Write the errors file every time there is an error so that if the | |
144 // user aborts the script, the error file is valid. | |
145 console.warn("error parsing result for " + type + " file= "+ file); | |
146 errorFiles.push(file); | |
147 fs.writeFileSync("output/errors.json", | |
148 JSON.stringify(errorFiles, null, ' '), 'utf8'); | |
149 } | |
150 onDone(); | |
151 }); | |
152 } | |
153 | |
154 var tasks = []; | |
155 | |
156 var numPending = numProcesses; | |
157 | |
158 function processNextTask() { | |
159 numPending--; | |
160 if (tasks.length > 0) { | |
161 numPending++; | |
162 var task = tasks.pop(); | |
163 task(); | |
164 } else { | |
165 if (numPending <= 0) { | |
166 console.log("Successfully completed all tasks"); | |
167 fs.writeFileSync("output/database.json", | |
168 JSON.stringify(db, null, ' '), 'utf8'); | |
169 } | |
170 } | |
171 } | |
172 | |
173 function createTask(type, entry, index) { | |
174 return function () { | |
175 var file = type + index + '.html'; | |
176 parseFile(type, processNextTask, entry, file, index); | |
177 }; | |
178 } | |
179 | |
180 for (var i = 0; i < domTypes.length; i++) { | |
181 var type = domTypes[i]; | |
182 var entries = cacheData[type]; | |
183 if (entries != null) { | |
184 for (var j = 0; j < entries.length; j++) { | |
185 tasks.push(createTask(type, entries[j], j)); | |
186 } | |
187 } else { | |
188 console.warn("No crawled files for " + type); | |
189 } | |
190 } | |
191 | |
192 for (var p = 0; p < numProcesses; p++) { | |
193 processNextTask(); | |
194 } | |
OLD | NEW |