DevTools: skip specific tests for backwards compatibility testing

third_party/WebKit/Source/devtools/scripts/npm_test.js

 // Copyright 2016 The Chromium Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
 var childProcess = require('child_process');
 var fs = require('fs');
 var path = require('path');
 var shell = require('child_process').execSync;
 
 var utils = require('./utils');
 
 var Flags = {
   DEBUG_DEVTOOLS: '--debug-devtools',
   DEBUG_DEVTOOLS_SHORTHAND: '-d',
   FETCH_CONTENT_SHELL: '--fetch-content-shell',
-  COMPAT_PROTOCOL: '--compat-protocol',
+  COMPAT_PROTOCOL: '--compat-protocol',  // backwards compatibility testing
+  CHROMIUM_PATH: '--chromium-path'       // useful for bisecting
 };
 
 var COMPAT_URL_MAPPING = {
   '1.2': 'https://storage.googleapis.com/content-shell-devtools-compat/content_shell_417361.zip',
 };
 
 var IS_DEBUG_ENABLED =
     utils.includes(process.argv, Flags.DEBUG_DEVTOOLS) || utils.includes(process.argv, Flags.DEBUG_DEVTOOLS_SHORTHAND);
 var COMPAT_PROTOCOL = utils.parseArgs(process.argv)[Flags.COMPAT_PROTOCOL];
+var CUSTOM_CHROMIUM_PATH = utils.parseArgs(process.argv)[Flags.CHROMIUM_PATH];
 var IS_FETCH_CONTENT_SHELL = utils.includes(process.argv, Flags.FETCH_CONTENT_SHELL);
 
 var CONTENT_SHELL_ZIP = 'content-shell.zip';
 var MAX_CONTENT_SHELLS = 10;
 var PLATFORM = getPlatform();
 var PYTHON = process.platform === 'win32' ? 'python.bat' : 'python';
 
-var CHROMIUM_SRC_PATH = path.resolve(__dirname, '..', '..', '..', '..', '..');
+var CHROMIUM_SRC_PATH = CUSTOM_CHROMIUM_PATH || path.resolve(__dirname, '..', '..', '..', '..', '..');
 var RELEASE_PATH = path.resolve(CHROMIUM_SRC_PATH, 'out', 'Release');
 var BLINK_TEST_PATH = path.resolve(CHROMIUM_SRC_PATH, 'blink', 'tools', 'run_layout_tests.py');
-var DEVTOOLS_PATH = path.resolve(__dirname, '..');
+var DEVTOOLS_PATH = path.resolve(CHROMIUM_SRC_PATH, 'third_party', 'WebKit', 'Source', 'devtools');
 var CACHE_PATH = path.resolve(DEVTOOLS_PATH, '.test_cache');
 var SOURCE_PATH = path.resolve(DEVTOOLS_PATH, 'front_end');
 
 function main() {
   if (!utils.isDir(CACHE_PATH))
     fs.mkdirSync(CACHE_PATH);
   deleteOldContentShells();
 
   if (COMPAT_PROTOCOL) {
     runCompatibilityTests();
@@ skipping 13 matching lines @@
 
   function onError(error) {
     console.log('Unable to run tests because of error:', error);
     console.log(`Try removing the .test_cache folder [${CACHE_PATH}] and retrying`);
   }
 }
 main();
 
 function runCompatibilityTests() {
   const folder = `compat-protocol-${COMPAT_PROTOCOL}`;
+  utils.removeRecursive(path.resolve(RELEASE_PATH, 'resources', 'inspector'));
   compileFrontend();
   var outPath = path.resolve(CACHE_PATH, folder, 'out');
   var contentShellDirPath = path.resolve(outPath, 'Release');
   var hasCachedContentShell = utils.isFile(getContentShellBinaryPath(contentShellDirPath));
   if (hasCachedContentShell) {
     console.log(`Using cached content shell at: ${outPath}`);
     copyFrontendToCompatBuildPath(contentShellDirPath, RELEASE_PATH);
     runTests(outPath, IS_DEBUG_ENABLED);
     return;
   }
   prepareContentShellDirectory(folder)
       .then(() => downloadContentShell(COMPAT_URL_MAPPING[COMPAT_PROTOCOL], folder))
       .then(extractContentShell)
       .then(() => copyFrontendToCompatBuildPath(contentShellDirPath, RELEASE_PATH))
       .then(() => runTests(outPath, IS_DEBUG_ENABLED));
 }
 
 function compileFrontend() {
   console.log('Compiling devtools frontend');
   try {
-    shell(`ninja -C ${RELEASE_PATH} devtools_frontend_resources`);
+    shell(`ninja -C ${RELEASE_PATH} devtools_frontend_resources`, {cwd: CHROMIUM_SRC_PATH});
   } catch (err) {
     console.log(err.stdout.toString());
     console.log('ERROR: Cannot compile frontend\n' + err);
     process.exit(1);
   }
 }
 
 function onUploadedCommitPosition(commitPosition) {
   var contentShellDirPath = path.resolve(CACHE_PATH, commitPosition, 'out', 'Release');
   var contentShellResourcesPath = path.resolve(contentShellDirPath, 'resources');
@@ skipping 163 matching lines @@
     '--no-pixel-tests',
     '--build-directory',
     buildDirectoryPath,
   ]);
   if (useDebugDevtools)
     testArgs.push('--additional-driver-flag=--debug-devtools');
   else
     console.log('TIP: You can debug a test using: npm run debug-test inspector/test-name.html');
 
   if (COMPAT_PROTOCOL) {
-    let platform = `protocol-${COMPAT_PROTOCOL}`;
-    let compatBaselinePath = path.resolve(DEVTOOLS_PATH, 'tests', 'baseline', platform);
+    const platform = `protocol-${COMPAT_PROTOCOL}`;
+    const testsPath = path.resolve(DEVTOOLS_PATH, 'tests');
+    const compatBaselinePath = path.resolve(testsPath, 'baseline', platform);
     testArgs.push(`--additional-platform-directory=${compatBaselinePath}`);
+    const expectationsPath = path.resolve(testsPath, 'TestExpectations');
+    testArgs.push(`--additional-expectations=${expectationsPath}`);
   }
   if (IS_DEBUG_ENABLED) {
     testArgs.push('--additional-driver-flag=--remote-debugging-port=9222');
     testArgs.push('--time-out-ms=6000000');
     console.log('\n=============================================');
     console.log('Go to: http://localhost:9222/');
     console.log('Click on link and in console execute: test()');
     console.log('=============================================\n');
   }
   var args = [BLINK_TEST_PATH].concat(testArgs).concat(getTestFlags());
@@ skipping 11 matching lines @@
 
 function getInspectorTests() {
   var specificTests = process.argv.filter(arg => utils.includes(arg, 'inspector'));
   if (specificTests.length)
     return specificTests;
   return [
     'inspector*',
     'http/tests/inspector*',
   ];
 }