gae.py: Remove support for MVMs (they are deprecated), simplify code a bit.

appengine/components/tool_support/gae_sdk_utils.py

 # Copyright 2013 The LUCI Authors. All rights reserved.
 # Use of this source code is governed under the Apache License, Version 2.0
 # that can be found in the LICENSE file.
 
 """Set of functions to work with GAE SDK tools."""
 
 import collections
 import glob
 import json
 import logging
@@ skipping 15 matching lines @@
 
 # Name of a directory with Go GAE SDK.
 GO_GAE_SDK = 'go_appengine'
 
 # Value of 'runtime: ...' in app.yaml -> SDK to use.
 RUNTIME_TO_SDK = {
   'go': GO_GAE_SDK,
   'python27': PYTHON_GAE_SDK,
 }
 
-# Exe name => instructions how to install it.
-KNOWN_TOOLS = {
-  'gcloud':
-      'Download and install the Google Cloud SDK from '
-      'https://cloud.google.com/sdk/',
-  'aedeploy':
-      'Install with: go install '
-      'google.golang.org/appengine/cmd/aedeploy',
-}
-
-
 # Path to a current SDK, set in setup_gae_sdk, accessible by gae_sdk_path.
 _GAE_SDK_PATH = None
 
 
-class BadEnvironmentConfig(Exception):
+class Error(Exception):
+  """Base class for a fatal error."""
+
+
+class BadEnvironmentError(Error):
   """Raised when required tools or environment are missing."""
 
 
+class UnsupportedModuleError(Error):
+  """Raised when trying to deploy MVM or Flex module."""
+
+
+class LoginRequiredError(Error):
+  """Raised by Application methods if use has to go through login flow."""
+
+
 def find_gae_sdk(sdk_name=PYTHON_GAE_SDK, search_dir=TOOLS_DIR):
   """Returns the path to GAE SDK if found, else None."""
   # First search up the directories up to root.
   while True:
     attempt = os.path.join(search_dir, sdk_name)
     if os.path.isfile(os.path.join(attempt, 'dev_appserver.py')):
       return attempt
     prev_dir = search_dir
     search_dir = os.path.dirname(search_dir)
     if search_dir == prev_dir:
@@ skipping 118 matching lines @@
   yaml = yaml_module
 
 
 def gae_sdk_path():
   """Checks that 'setup_gae_sdk' was called and returns a path to GAE SDK."""
   if not _GAE_SDK_PATH:
     raise ValueError('setup_gae_sdk wasn\'t called')
   return _GAE_SDK_PATH
 
 
-class LoginRequiredError(Exception):
-  """Raised by Application methods if use has to go through login flow."""
-
-
 ModuleFile = collections.namedtuple('ModuleFile', ['path', 'data'])
 
 
 class Application(object):
   """Configurable GAE application.
 
   Can be used to query and change GAE application configuration (default
   serving version, uploaded versions, etc.). Built on top of appcfg.py calls.
   """
 
@@ skipping 69 matching lines @@
     cmd = [
       sys.executable,
       os.path.join(gae_sdk_path(), 'appcfg.py'),
       '--application', self.app_id,
       '--noauth_local_webserver',
       'list_versions',
     ]
     return subprocess.call(cmd, cwd=self._app_dir) == 0
 
   def run_cmd(self, cmd, cwd=None):
-    """Runs subprocess, capturing the output."""
+    """Runs subprocess, capturing the output.
+
+    Doesn't close stdin, since gcloud may me asking for user input. If this is

[Vadim Sh., 2017/05/23 19:36:11]

this is actually needed for http://shortn/_h5Bh8n3Y5c and there's no workaround
currently (this thing is internal, please don't leak details in the CL).

I hope at some point it won't be needed...

[iannucci, 2017/05/23 19:38:08]

s/may me/may be

[Vadim Sh., 2017/05/23 19:46:36]

Done.

+    undesirable (e.g when gae.py is used from scripts), close 'stdin' of gae.py
+    process itself.
+    """
     logging.debug('Running %s', cmd)
     proc = subprocess.Popen(
         cmd,
         cwd=cwd or self._app_dir,
-        stdout=subprocess.PIPE,
-        stdin=subprocess.PIPE)
-    output, _ = proc.communicate(None)
+        stdout=subprocess.PIPE)
+    output, _ = proc.communicate()
     if proc.returncode:
       sys.stderr.write('\n' + output + '\n')
       raise subprocess.CalledProcessError(proc.returncode, cmd, output)
     return output
 
   def run_appcfg(self, args):
     """Runs appcfg.py <args>, deserializes its output and returns it."""
     if not is_oauth_token_cached():
       raise LoginRequiredError('Login first using \'login\' subcommand.')
     cmd = [
       sys.executable,
       os.path.join(gae_sdk_path(), 'appcfg.py'),
       '--application', self.app_id,
     ]
     if self._verbose:
       cmd.append('--verbose')
     cmd.extend(args)
     return yaml.safe_load(self.run_cmd(cmd))
 
   def run_gcloud(self, args):
     """Runs gcloud <args>."""
+    gcloud = find_gcloud()
     if not is_gcloud_oauth2_token_cached():
       raise LoginRequiredError('Login first using \'gcloud auth login\'')
-    check_tool_in_path('gcloud')
-    return self.run_cmd(['gcloud'] + args)
+    return self.run_cmd([gcloud] + args)
 
   def list_versions(self):
     """List all uploaded versions.
 
     Returns:
       Dict {module name -> [list of uploaded versions]}.
     """
     return self.run_appcfg(['list_versions'])
 
   def set_default_version(self, version, modules=None):
@@ skipping 13 matching lines @@
         'delete_version',
         '--module', module,
         '--version', version,
       ])
 
   def update_modules(self, version, modules=None):
     """Deploys new version of the given module names.
 
     Supports deploying modules both with Managed VMs and AppEngine v1 runtime.
     """
-    reg_modules = []
-    mvm_modules = []
+    mods = []
     try:
       for m in sorted(modules or self.modules):
         mod = self._modules[m]
         if mod.data.get('vm'):
-          mvm_modules.append(mod)
-        else:
-          reg_modules.append(mod)
+          raise UnsupportedModuleError('MVM is not supported: %s' % m)
+        if mod.data.get('env') == 'flex':
+          raise UnsupportedModuleError('Flex is not supported yet: %s' % m)
         if mod.data.get('runtime') == 'go' and not os.environ.get('GOROOT'):
-          raise BadEnvironmentConfig('GOROOT must be set when deploying Go app')
+          raise BadEnvironmentError('GOROOT must be set when deploying Go app')
+        mods.append(mod)
     except KeyError as e:
       raise ValueError('Unknown module: %s' % e)
-    if reg_modules:
-      # Always make 'default' the first module to be uploaded.
-      reg_modules.sort(key=lambda x: '' if x == 'default' else x)
-      self.run_appcfg(
-          ['update'] + [m.path for m in reg_modules] + ['--version', version])
-    # Go modules have to be deployed one at a time, based on docs.
-    for m in mvm_modules:
-      self.deploy_mvm_module(m, version)
+    # Always make 'default' the first module to be uploaded.
+    mods.sort(key=lambda x: '' if x == 'default' else x)
+    self.run_appcfg(
+        ['update'] + [m.path for m in mods] + ['--version', version])
 
   def update_indexes(self):
     """Deploys new index.yaml."""
     if os.path.isfile(os.path.join(self.default_module_dir, 'index.yaml')):
       self.run_appcfg(['update_indexes', self.default_module_dir])
 
   def update_queues(self):
     """Deploys new queue.yaml."""
     if os.path.isfile(os.path.join(self.default_module_dir, 'queue.yaml')):
       self.run_appcfg(['update_queues', self.default_module_dir])
@@ skipping 68 matching lines @@
 
     # Sort by version number (best effort, nonconforming version names will
     # appear first in the list).
     def extract_version_num(version):
       try:
         return int(version.split('-', 1)[0])
       except ValueError:
         return -1
     return sorted(actual_versions, key=extract_version_num)
 
-  def deploy_mvm_module(self, mod, version):
-    """Uses gcloud to upload MVM module using remote docker build.
-
-    Assumes 'gcloud' and 'aedeploy' are in PATH.
-    """
-    # 'aedeploy' requires cwd to be set to module path.
-    check_tool_in_path('gcloud')
-    cmd = [
-      'gcloud', 'app', 'deploy', os.path.basename(mod.path),
-      '--project', self.app_id,
-      '--version', version,
-      '--docker-build', 'remote',
-      '--no-promote', '--force',
-    ]
-    if self._verbose:
-      cmd.extend(['--verbosity', 'debug'])
-    if mod.data.get('runtime') == 'go':
-      check_tool_in_path('aedeploy')
-      cmd = ['aedeploy'] + cmd
-    self.run_cmd(cmd, cwd=os.path.dirname(mod.path))
-
   def get_actives(self, modules=None):
     """Returns active version(s)."""
     args = [
       'app', 'versions', 'list',
       '--project', self.app_id,
       '--format', 'json',
       '--hide-no-traffic',
     ]
     raw = self.run_gcloud(args)
     try:
       data = json.loads(raw)
     except ValueError:
       sys.stderr.write('Failed to decode %r as JSON\n' % raw)
       raise
     # TODO(maruel): Handle when traffic_split != 1.0.
     # TODO(maruel): There's a lot more data, decide what is generally useful in
     # there.
     return [
       {
         'creationTime': service['version']['createTime'],
         'deployer': service['version']['createdBy'],
         'id': service['id'],
         'service': service['service'],
       } for service in data if not modules or service['service'] in modules
     ]
 
 
-def check_tool_in_path(tool):
-  """Raises BadEnvironmentConfig error if no such executable in PATH."""
+def find_gcloud():
+  """Searches for 'gcloud' binary in PATH and returns absolute path to it.
+
+  Raises BadEnvironmentError error if it's not there.
+  """
   for path in os.environ['PATH'].split(os.pathsep):
-    exe_file = os.path.join(path, tool)
+    exe_file = os.path.join(path, 'gcloud')
     if os.path.isfile(exe_file) and os.access(exe_file, os.X_OK):
-      return
-  msg = 'Can\'t find "%s" in PATH.' % tool
-  if tool in KNOWN_TOOLS:
-    msg += ' ' + KNOWN_TOOLS[tool]
-  raise BadEnvironmentConfig(msg)
+      return exe_file
+  raise BadEnvironmentError(
+      'Can\'t find "gcloud" in PATH. Install the Google Cloud SDK from '
+      'https://cloud.google.com/sdk/')
 
 
 def setup_env(app_dir, app_id, version, module_id, remote_api=False):
   """Setups os.environ so GAE code works."""
   # GCS library behaves differently when running under remote_api. It uses
   # SERVER_SOFTWARE to figure this out. See cloudstorage/common.py, local_run().
   if remote_api:
     os.environ['SERVER_SOFTWARE'] = 'remote_api'
   else:
     os.environ['SERVER_SOFTWARE'] = 'Development yo dawg/1.0'
@@ skipping 42 matching lines @@
     New instance of Application configured based on passed options.
   """
   logging.basicConfig(level=logging.DEBUG if options.verbose else logging.ERROR)
 
   if not app_dir and not options.app_dir:
     parser.error('--app-dir option is required')
   app_dir = os.path.abspath(app_dir or options.app_dir)
 
   try:
     runtime = get_app_runtime(find_app_yamls(app_dir))
-  except (BadEnvironmentConfig, ValueError) as exc:
+  except (Error, ValueError) as exc:
     parser.error(str(exc))
 
   sdk_path = options.sdk_path or find_gae_sdk(RUNTIME_TO_SDK[runtime], app_dir)
   if not sdk_path:
     parser.error('Failed to find the AppEngine SDK. Pass --sdk-path argument.')
 
   setup_gae_sdk(sdk_path)
 
   try:
     return Application(app_dir, options.app_id, options.verbose)
-  except (BadEnvironmentConfig, ValueError) as e:
+  except (Error, ValueError) as e:
     parser.error(str(e))
 
 
 def confirm(text, app, version, modules=None, default_yes=False):
   """Asks a user to confirm the action related to GAE app.
 
   Args:
     text: actual text of the prompt.
     app: instance of Application.
     version: version or a list of versions to operate upon.
@@ skipping 26 matching lines @@
     with open(p) as f:
       return len(json.load(f)['data']) != 0
   except (KeyError, IOError, OSError, ValueError):
     return False
 
 
 def setup_gae_env():
   """Sets up App Engine Python test environment."""
   sdk_path = find_gae_sdk(PYTHON_GAE_SDK)
   if not sdk_path:
-    raise BadEnvironmentConfig('Couldn\'t find GAE SDK.')
+    raise BadEnvironmentError('Couldn\'t find GAE SDK.')
   setup_gae_sdk(sdk_path)