Use logdog butler subcommand to run tests.

tools/mb/mb.py

 #!/usr/bin/env python
 # Copyright 2015 The Chromium Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
 """MB - the Meta-Build wrapper around GYP and GN
 
 MB is a wrapper script for GYP and GN that can be used to generate build files
 for sets of canned configurations and analyze them.
 """
 
 from __future__ import print_function
 
 import argparse
 import ast
 import errno
 import json
+import logging
 import os
 import pipes
 import pprint
 import re
 import shutil
 import sys
 import subprocess
 import tempfile
 import traceback
 import urllib2
@@ skipping 1051 matching lines @@
     executable_suffix = '.exe' if self.platform == 'win32' else ''
 
     cmdline = []
     extra_files = []
 
     if test_type == 'nontest':
       self.WriteFailureAndRaise('We should not be isolating %s.' % target,
                                 output_path=None)
 
     if android and test_type != "script":
-      logdog_command = [
-          '--logdog-bin-cmd', './../../bin/logdog_butler',
-          '--project', 'chromium',
-          '--service-account-json',
-          '/creds/service_accounts/service-account-luci-logdog-publisher.json',
-          '--prefix', 'android/swarming/logcats/${SWARMING_TASK_ID}',
-          '--source', '${ISOLATED_OUTDIR}/logcats',
-          '--name', 'unified_logcats',
-      ]
       test_cmdline = [
           self.PathJoin('bin', 'run_%s' % target),
           '--logcat-output-file', '${ISOLATED_OUTDIR}/logcats',
           '--target-devices-file', '${SWARMING_BOT_FILE}',
           '-v'
       ]
-      cmdline = (['./../../build/android/test_wrapper/logdog_wrapper.py']
-                 + logdog_command + test_cmdline)
+      if not os.path.exists('./../../bin/logdog_butler'):

[jbudorick, 2017/02/17 01:46:21]

This check doesn't work as intended, as mb.py runs on a different bot than the
task itself.

[dnj, 2017/02/17 16:54:52]

Make this a constant (e.g., "butler_path") so you can reference it later instead
of repeating the relative pathing. Also, I'd recommend resolving this to
abspath.

[BigBossZhiling, 2017/02/17 21:10:41]

Acknowledged.

+        logging.error(
+            'Logdog binary %s unavailable. Unable to upload logcats.',

[dnj, 2017/02/17 16:54:52]

Should this maybe be fatal? If something is putting the file there, and it's
randomly not there, then it seems like something went horribly wrong.

[BigBossZhiling, 2017/02/17 21:10:41]

Acknowledged.

+            './../../bin/logdog_butler')
+        cmdline = test_cmdline
+      else:
+        project = 'chromium'
+        output = 'logdog,host=services-dot-luci-logdog.appspot.com'
+        prefix = 'android/swarming/logcats/${SWARMING_TASK_ID}'

[dnj, 2017/02/17 16:54:52]

Does the shell actually resolve this? Crazy. I still recommend resolving this in
Python (e.g., os.environ['SWARMING_TASK_ID']) so it's clear and not
shell-reliant.

[BigBossZhiling, 2017/02/17 21:10:41]

Done.

+        service_account_json = '/creds/service_accounts/'\

[dnj, 2017/02/17 16:54:51]

nit: recommend using parenthesis for breaking:

service_account_json = ('creds/service_acconts/'
  'service-...')

That way you don't need the line continuation.

[BigBossZhiling, 2017/02/17 21:10:41]

Done.

+            'service-account-luci-logdog-publisher.json'
+
+        # Call test_cmdline through logdog butler subcommand.
+        test_cmdline = [
+            './../../bin/logdog_butler', '-project', project,
+            '-output', output,
+            '-prefix', prefix,
+            '-service-account-json', service_account_json,

[dnj, 2017/02/17 16:54:51]

Not sure what this is supposed to accomplish. You don't connect any
STDOUT/STDERR, and you don't have any streamserver-uri defined, so this is
basically just a subcommand call as-is.

[BigBossZhiling, 2017/02/17 21:10:41]

Acknowledged.

+            'run', '--'] + test_cmdline
+
+        if not os.path.exists('${ISOLATED_OUTDIR}/logcats'):

[mikecase (-- gone --), 2017/02/17 01:40:57]

I think we want to upload the logcats via the test_runner. So, like, add a line
(maybe in b/a/test_runner.py) that is like, if --upload-full-logcat record the
full logcat and upload it.

Will help with kinda consolidating where in the code we upload things to logdog.

[jbudorick, 2017/02/17 01:46:21]

Everything below this shouldn't be here.

[BigBossZhiling, 2017/02/17 21:10:41]

Done.

+          logging.error(
+              'Logcat sources not found at %s. Unable to upload logcats.',
+              '${ISOLATED_OUTDIR}/logcats')
+          cmdline = test_cmdline
+        else:
+          stream_source_cmdline = [
+              './../../bin/logdog_butler', '-project', project,

[mikecase (-- gone --), 2017/02/17 01:40:57]

nit: Can remove the "./" at the beginning of the path.

[BigBossZhiling, 2017/02/17 21:10:41]

Done.

+              '-output', output,
+              '-prefix', prefix,
+              '-service-account-json', service_account_json,
+              'stream', '-source', '${ISOLATED_OUTDIR}/logcats',
+              '-stream', '-name=unified_logcats']
+          cmdline = test_cmdline + ['&&'] + stream_source_cmdline
     elif use_xvfb and test_type == 'windowed_test_launcher':
       extra_files = [
           '../../testing/test_env.py',
           '../../testing/xvfb.py',
       ]
       cmdline = [
         '../../testing/xvfb.py',
         './' + str(executable) + executable_suffix,
         '--brave-new-test-launcher',
         '--test-launcher-bot-mode',
@@ skipping 453 matching lines @@
   # Then check to see if the arg contains any metacharacters other than
   # double quotes; if it does, quote everything (including the double
   # quotes) for safety.
   if any(a in UNSAFE_FOR_CMD for a in arg):
     arg = ''.join('^' + a if a in ALL_META_CHARS else a for a in arg)
   return arg
 
 
 if __name__ == '__main__':
   sys.exit(main(sys.argv[1:]))