Use [CACHE]/cipd/goma for goma_dir in recipes/chromium_codesearch.py

scripts/slave/recipes/chromium_codesearch.expected/full_ChromiumOS_Codesearch_fail.json

 [
   {
     "cmd": [
       "python",
       "-u",
       "RECIPE_MODULE[depot_tools::bot_update]/resources/bot_update.py",
       "--spec",
       "cache_dir = '[GIT_CACHE]'\nsolutions = [{'deps_file': '.DEPS.git', 'managed': True, 'name': 'src', 'url': 'https://chromium.googlesource.com/chromium/src.git'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/chrome-devtools-frontend', 'url': 'https://chromium.googlesource.com/chromium/tools/chrome-devtools-frontend'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/gsd_generate_index', 'url': 'https://chromium.googlesource.com/chromium/tools/gsd_generate_index'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/perf', 'url': 'https://chromium.googlesource.com/chromium/tools/perf'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/command_wrapper/bin', 'url': 'https://chromium.googlesource.com/chromium/tools/command_wrapper/bin'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/depot_tools', 'url': 'https://chromium.googlesource.com/chromium/tools/depot_tools'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/chromium-jobqueue', 'url': 'https://chromium.googlesource.com/chromium/tools/chromium-jobqueue'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/chromium-shortener', 'url': 'https://chromium.googlesource.com/chromium/tools/chromium-shortener'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'tools/deps2git', 'url': 'https://chromium.googlesource.com/chromium/tools/deps2git'}, {'deps_file': '.DEPS.git', 'managed': True, 'name': 'infra', 'url': 'https://chromium.googlesource.com/infra/infra'}]",
       "--patch_root",
       "src",
@@ skipping 99 matching lines @@
       "@@@STEP_LOG_LINE@python.inline@import os, sys@@@",
       "@@@STEP_LOG_LINE@python.inline@from common import chromium_utils # Error? See https://crbug.com/584783.@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
       "@@@STEP_LOG_LINE@python.inline@@@@",
       "@@@STEP_LOG_LINE@python.inline@if os.path.exists(sys.argv[1]):@@@",
       "@@@STEP_LOG_LINE@python.inline@  chromium_utils.RemoveDirectory(sys.argv[1])@@@",
       "@@@STEP_LOG_END@python.inline@@@"
     ]
   },
   {
+    "cmd": [],
+    "name": "ensure_goma"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[depot_tools::cipd]/resources/bootstrap.py",
+      "--platform",
+      "linux-amd64",
+      "--dest-directory",
+      "[SLAVE_BUILD]/cipd",
+      "--json-output",
+      "/path/to/tmp/json"
+    ],
+    "name": "ensure_goma.install cipd",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_TEXT@cipd instance_id: 40-chars-fake-of-the-package-instance_id@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"executable\": \"[SLAVE_BUILD]/cipd/cipd\", @@@",
+      "@@@STEP_LOG_LINE@json.output@  \"instance_id\": \"40-chars-fake-of-the-package-instance_id\"@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "[SLAVE_BUILD]/cipd/cipd",
+      "ensure",
+      "--root",
+      "[CACHE]/cipd/goma",
+      "--list",
+      "infra/tools/cloudtail/linux-amd64 goma_recipe_module\ninfra_internal/goma/client/linux-amd64 release",
+      "--json-output",
+      "/path/to/tmp/json",
+      "--service-account-json",
+      "/creds/service_accounts/service-account-goma-client.json"
+    ],
+    "name": "ensure_goma.ensure_installed",
+    "~followup_annotations": [
+      "@@@STEP_NEST_LEVEL@1@@@",
+      "@@@STEP_LOG_LINE@json.output@{@@@",
+      "@@@STEP_LOG_LINE@json.output@  \"result\": [@@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-goma_recipe_modu\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"infra/tools/cloudtail/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }, @@@",
+      "@@@STEP_LOG_LINE@json.output@    {@@@",
+      "@@@STEP_LOG_LINE@json.output@      \"instance_id\": \"resolved-instance_id-of-release---------\", @@@",
+      "@@@STEP_LOG_LINE@json.output@      \"package\": \"infra_internal/goma/client/linux-amd64\"@@@",
+      "@@@STEP_LOG_LINE@json.output@    }@@@",
+      "@@@STEP_LOG_LINE@json.output@  ]@@@",
+      "@@@STEP_LOG_LINE@json.output@}@@@",
+      "@@@STEP_LOG_END@json.output@@@"
+    ]
+  },
+  {
     "cmd": [
       "python",
       "-u",
       "RECIPE_PACKAGE_REPO[depot_tools]/gclient.py",
       "runhooks"
     ],
     "cwd": "[SLAVE_BUILD]/src",
     "env": {
       "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json",
-      "GYP_DEFINES": "clang=1 component=shared_library fastbuild=1 gomadir='[BUILD]/goma' target_arch=x64 use_goma=1",
+      "GYP_DEFINES": "clang=1 component=shared_library fastbuild=1 gomadir='[CACHE]/cipd/goma' target_arch=x64 use_goma=1",
       "PATH": "%(PATH)s:RECIPE_PACKAGE_REPO[depot_tools]"
     },
     "name": "gclient runhooks"
   },
   {
     "cmd": [
       "[SLAVE_BUILD]/src/buildtools/linux64/gn",
       "gen",
       "[SLAVE_BUILD]/src/out/Debug",
-      "--args=is_clang=true is_component_build=true is_debug=true goma_dir=\"/b/build/goma\" symbol_level=1 target_cpu=\"x64\" use_goma=true target_os=\"chromeos\" use_ozone=true"
+      "--args=is_clang=true is_component_build=true is_debug=true symbol_level=1 target_cpu=\"x64\" target_os=\"chromeos\" use_ozone=true use_goma=true goma_dir=[CACHE]/cipd/goma"
     ],
     "cwd": "[SLAVE_BUILD]/src",
     "name": "generate build files for chromeos"
   },
   {
     "cmd": [
+      "python",
+      "-u",
+      "\nimport multiprocessing\nimport sys\n\njob_limit = 200\nif sys.platform.startswith('linux'):\n  # Use 80 for linux not to load goma backend.\n  job_limit = 80\n\ntry:\n  jobs = min(job_limit, multiprocessing.cpu_count() * 10)\nexcept NotImplementedError:\n  jobs = 50\n\nprint jobs\n"
+    ],
+    "name": "calculate the number of recommended jobs",
+    "stdout": "/path/to/tmp/",
+    "~followup_annotations": [
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@import multiprocessing@@@",
+      "@@@STEP_LOG_LINE@python.inline@import sys@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@job_limit = 200@@@",
+      "@@@STEP_LOG_LINE@python.inline@if sys.platform.startswith('linux'):@@@",
+      "@@@STEP_LOG_LINE@python.inline@  # Use 80 for linux not to load goma backend.@@@",
+      "@@@STEP_LOG_LINE@python.inline@  job_limit = 80@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@try:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = min(job_limit, multiprocessing.cpu_count() * 10)@@@",
+      "@@@STEP_LOG_LINE@python.inline@except NotImplementedError:@@@",
+      "@@@STEP_LOG_LINE@python.inline@  jobs = 50@@@",
+      "@@@STEP_LOG_LINE@python.inline@@@@",
+      "@@@STEP_LOG_LINE@python.inline@print jobs@@@",
+      "@@@STEP_LOG_END@python.inline@@@"
+    ]
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "restart"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "start_goma"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::goma]/resources/cloudtail_utils.py",
+      "start",
+      "--cloudtail-path",
+      "[CACHE]/cipd/goma/cloudtail"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "start cloudtail",
+    "stdout": "/path/to/tmp/"
+  },
+  {
+    "cmd": [
       "ninja",
       "-C",
       "[SLAVE_BUILD]/src/out/Debug",
       "All",
       "-t",
       "compdb",
       "cc",
       "cxx",
       "objc",
-      "objcxx"
+      "objcxx",
+      "-j",
+      "50"
     ],
     "name": "generate compilation database for chromeos",
     "stdout": "/path/to/tmp/"
   },
   {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "jsonstatus",
+      "[CACHE]/cipd/goma/jsonstatus"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "goma_jsonstatus"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "stat"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "goma_stat"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "stop"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "stop_goma"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE_REPO[build]/scripts/slave/upload_goma_logs.py",
+      "--upload-compiler-proxy-info",
+      "--json-status",
+      "[CACHE]/cipd/goma/jsonstatus",
+      "--ninja-log-outdir",
+      "[SLAVE_BUILD]/src/out/Debug",
+      "--ninja-log-compiler",
+      "goma",
+      "--ninja-log-command",
+      "['ninja', '-C', Path('checkout', 'out','Debug'), 'All', '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx', '-j', 50]",
+      "--ninja-log-exit-status",
+      "0",
+      "--buildbot-buildername",
+      "ChromiumOS Codesearch",
+      "--buildbot-mastername",
+      "chromium.infra.cron",
+      "--buildbot-slavename",
+      "TestSlavename"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "upload_log"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::goma]/resources/cloudtail_utils.py",
+      "stop",
+      "--killed-pid",
+      "12345"
+    ],
+    "name": "stop cloudtail"
+  },
+  {
     "allow_subannotations": true,
     "cmd": [
       "python",
       "-u",
       "RECIPE_PACKAGE_REPO[build]/scripts/slave/clang_revision.py",
       "--src-dir",
       "[SLAVE_BUILD]/src",
       "--output-json",
       "/path/to/tmp/json"
     ],
@@ skipping 34 matching lines @@
       "TestSlavename",
       "--build-args",
       "-k 0",
       "--compiler",
       "goma-clang",
       "--goma-jsonstatus",
       "/path/to/tmp/json",
       "--goma-service-account-json-file",
       "/creds/service_accounts/service-account-goma-client.json",
       "--goma-dir",
-      "[BUILD]/goma",
+      "[CACHE]/cipd/goma",
       "--",
       "All"
     ],
     "env": {
       "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
     },
     "name": "compile",
     "~followup_annotations": [
       "@@@STEP_LOG_LINE@json.output (invalid)@null@@@",
       "@@@STEP_LOG_END@json.output (invalid)@@@"
     ]
   },
   {
     "cmd": [
       "cp",
       "some compilation data",
       "[SLAVE_BUILD]/src/out/Debug/compile_commands.json"
     ],
     "name": "copy compilation database"
   },
   {
     "cmd": [
       "[SLAVE_BUILD]/src/buildtools/linux64/gn",
       "gen",
       "[SLAVE_BUILD]/src/out/Debug",
-      "--args=is_clang=true is_component_build=true is_debug=true goma_dir=\"/b/build/goma\" symbol_level=1 target_cpu=\"x64\" use_goma=true"
+      "--args=is_clang=true is_component_build=true is_debug=true symbol_level=1 target_cpu=\"x64\" use_goma=true goma_dir=[CACHE]/cipd/goma"
     ],
     "cwd": "[SLAVE_BUILD]/src",
     "name": "generate build files for linux"
   },
   {
     "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "restart"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "start_goma (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::goma]/resources/cloudtail_utils.py",
+      "start",
+      "--cloudtail-path",
+      "[CACHE]/cipd/goma/cloudtail"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "start cloudtail (2)",
+    "stdout": "/path/to/tmp/"
+  },
+  {
+    "cmd": [
       "ninja",
       "-C",
       "[SLAVE_BUILD]/src/out/Debug",
       "All",
       "-t",
       "compdb",
       "cc",
       "cxx",
       "objc",
-      "objcxx"
+      "objcxx",
+      "-j",
+      "50"
     ],
     "name": "generate compilation database for linux",
     "stdout": "/path/to/tmp/"
   },
   {
     "cmd": [
       "python",
       "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "jsonstatus",
+      "[CACHE]/cipd/goma/jsonstatus"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "goma_jsonstatus (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "stat"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "goma_stat (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "[CACHE]/cipd/goma/goma_ctl.py",
+      "stop"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "stop_goma (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_PACKAGE_REPO[build]/scripts/slave/upload_goma_logs.py",
+      "--upload-compiler-proxy-info",
+      "--json-status",
+      "[CACHE]/cipd/goma/jsonstatus",
+      "--ninja-log-outdir",
+      "[SLAVE_BUILD]/src/out/Debug",
+      "--ninja-log-compiler",
+      "goma",
+      "--ninja-log-command",
+      "['ninja', '-C', Path('checkout', 'out','Debug'), 'All', '-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx', '-j', 50]",
+      "--ninja-log-exit-status",
+      "0",
+      "--buildbot-buildername",
+      "ChromiumOS Codesearch",
+      "--buildbot-mastername",
+      "chromium.infra.cron",
+      "--buildbot-slavename",
+      "TestSlavename"
+    ],
+    "env": {
+      "GOMA_SERVICE_ACCOUNT_JSON_FILE": "/creds/service_accounts/service-account-goma-client.json"
+    },
+    "name": "upload_log (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
+      "RECIPE_MODULE[build::goma]/resources/cloudtail_utils.py",
+      "stop",
+      "--killed-pid",
+      "12345"
+    ],
+    "name": "stop cloudtail (2)"
+  },
+  {
+    "cmd": [
+      "python",
+      "-u",
       "[BUILD]/scripts/slave/chromium/filter_compilations.py",
       "--compdb-input",
       "[SLAVE_BUILD]/src/out/Debug/compile_commands.json",
       "--compdb-filter",
       "some compilation data",
       "--compdb-output",
       "[SLAVE_BUILD]/src/out/Debug/compile_commands.json"
     ],
     "name": "Filter out duplicate compilation units"
   },
@@ skipping 83 matching lines @@
     "~followup_annotations": [
       "@@@STEP_LINK@gsutil.upload@https://storage.cloud.google.com/chrome-codesearch/prod/chromium_src_chromeos_170242.tar.bz2@@@"
     ]
   },
   {
     "name": "$result",
     "recipe_result": null,
     "status_code": 0
   }
 ]