Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(438)

Unified Diff: tools/telemetry/third_party/gsutil/third_party/boto/boto/emr/step.py

Issue 1260493004: Revert "Add gsutil 4.13 to telemetry/third_party" (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 5 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
Index: tools/telemetry/third_party/gsutil/third_party/boto/boto/emr/step.py
diff --git a/tools/telemetry/third_party/gsutil/third_party/boto/boto/emr/step.py b/tools/telemetry/third_party/gsutil/third_party/boto/boto/emr/step.py
deleted file mode 100644
index de6835fb4ebdacaa0328e7a1d562b4820eb77c9d..0000000000000000000000000000000000000000
--- a/tools/telemetry/third_party/gsutil/third_party/boto/boto/emr/step.py
+++ /dev/null
@@ -1,283 +0,0 @@
-# Copyright (c) 2010 Spotify AB
-# Copyright (c) 2010-2011 Yelp
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-from boto.compat import six
-
-
-class Step(object):
- """
- Jobflow Step base class
- """
- def jar(self):
- """
- :rtype: str
- :return: URI to the jar
- """
- raise NotImplemented()
-
- def args(self):
- """
- :rtype: list(str)
- :return: List of arguments for the step
- """
- raise NotImplemented()
-
- def main_class(self):
- """
- :rtype: str
- :return: The main class name
- """
- raise NotImplemented()
-
-
-class JarStep(Step):
- """
- Custom jar step
- """
- def __init__(self, name, jar, main_class=None,
- action_on_failure='TERMINATE_JOB_FLOW', step_args=None):
- """
- A elastic mapreduce step that executes a jar
-
- :type name: str
- :param name: The name of the step
- :type jar: str
- :param jar: S3 URI to the Jar file
- :type main_class: str
- :param main_class: The class to execute in the jar
- :type action_on_failure: str
- :param action_on_failure: An action, defined in the EMR docs to
- take on failure.
- :type step_args: list(str)
- :param step_args: A list of arguments to pass to the step
- """
- self.name = name
- self._jar = jar
- self._main_class = main_class
- self.action_on_failure = action_on_failure
-
- if isinstance(step_args, six.string_types):
- step_args = [step_args]
-
- self.step_args = step_args
-
- def jar(self):
- return self._jar
-
- def args(self):
- args = []
-
- if self.step_args:
- args.extend(self.step_args)
-
- return args
-
- def main_class(self):
- return self._main_class
-
-
-class StreamingStep(Step):
- """
- Hadoop streaming step
- """
- def __init__(self, name, mapper, reducer=None, combiner=None,
- action_on_failure='TERMINATE_JOB_FLOW',
- cache_files=None, cache_archives=None,
- step_args=None, input=None, output=None,
- jar='/home/hadoop/contrib/streaming/hadoop-streaming.jar'):
- """
- A hadoop streaming elastic mapreduce step
-
- :type name: str
- :param name: The name of the step
- :type mapper: str
- :param mapper: The mapper URI
- :type reducer: str
- :param reducer: The reducer URI
- :type combiner: str
- :param combiner: The combiner URI. Only works for Hadoop 0.20
- and later!
- :type action_on_failure: str
- :param action_on_failure: An action, defined in the EMR docs to
- take on failure.
- :type cache_files: list(str)
- :param cache_files: A list of cache files to be bundled with the job
- :type cache_archives: list(str)
- :param cache_archives: A list of jar archives to be bundled with
- the job
- :type step_args: list(str)
- :param step_args: A list of arguments to pass to the step
- :type input: str or a list of str
- :param input: The input uri
- :type output: str
- :param output: The output uri
- :type jar: str
- :param jar: The hadoop streaming jar. This can be either a local
- path on the master node, or an s3:// URI.
- """
- self.name = name
- self.mapper = mapper
- self.reducer = reducer
- self.combiner = combiner
- self.action_on_failure = action_on_failure
- self.cache_files = cache_files
- self.cache_archives = cache_archives
- self.input = input
- self.output = output
- self._jar = jar
-
- if isinstance(step_args, six.string_types):
- step_args = [step_args]
-
- self.step_args = step_args
-
- def jar(self):
- return self._jar
-
- def main_class(self):
- return None
-
- def args(self):
- args = []
-
- # put extra args BEFORE -mapper and -reducer so that e.g. -libjar
- # will work
- if self.step_args:
- args.extend(self.step_args)
-
- args.extend(['-mapper', self.mapper])
-
- if self.combiner:
- args.extend(['-combiner', self.combiner])
-
- if self.reducer:
- args.extend(['-reducer', self.reducer])
- else:
- args.extend(['-jobconf', 'mapred.reduce.tasks=0'])
-
- if self.input:
- if isinstance(self.input, list):
- for input in self.input:
- args.extend(('-input', input))
- else:
- args.extend(('-input', self.input))
- if self.output:
- args.extend(('-output', self.output))
-
- if self.cache_files:
- for cache_file in self.cache_files:
- args.extend(('-cacheFile', cache_file))
-
- if self.cache_archives:
- for cache_archive in self.cache_archives:
- args.extend(('-cacheArchive', cache_archive))
-
- return args
-
- def __repr__(self):
- return '%s.%s(name=%r, mapper=%r, reducer=%r, action_on_failure=%r, cache_files=%r, cache_archives=%r, step_args=%r, input=%r, output=%r, jar=%r)' % (
- self.__class__.__module__, self.__class__.__name__,
- self.name, self.mapper, self.reducer, self.action_on_failure,
- self.cache_files, self.cache_archives, self.step_args,
- self.input, self.output, self._jar)
-
-
-class ScriptRunnerStep(JarStep):
-
- ScriptRunnerJar = 's3n://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar'
-
- def __init__(self, name, **kw):
- super(ScriptRunnerStep, self).__init__(name, self.ScriptRunnerJar, **kw)
-
-
-class PigBase(ScriptRunnerStep):
-
- BaseArgs = ['s3n://us-east-1.elasticmapreduce/libs/pig/pig-script',
- '--base-path', 's3n://us-east-1.elasticmapreduce/libs/pig/']
-
-
-class InstallPigStep(PigBase):
- """
- Install pig on emr step
- """
-
- InstallPigName = 'Install Pig'
-
- def __init__(self, pig_versions='latest'):
- step_args = []
- step_args.extend(self.BaseArgs)
- step_args.extend(['--install-pig'])
- step_args.extend(['--pig-versions', pig_versions])
- super(InstallPigStep, self).__init__(self.InstallPigName, step_args=step_args)
-
-
-class PigStep(PigBase):
- """
- Pig script step
- """
-
- def __init__(self, name, pig_file, pig_versions='latest', pig_args=[]):
- step_args = []
- step_args.extend(self.BaseArgs)
- step_args.extend(['--pig-versions', pig_versions])
- step_args.extend(['--run-pig-script', '--args', '-f', pig_file])
- step_args.extend(pig_args)
- super(PigStep, self).__init__(name, step_args=step_args)
-
-
-class HiveBase(ScriptRunnerStep):
-
- BaseArgs = ['s3n://us-east-1.elasticmapreduce/libs/hive/hive-script',
- '--base-path', 's3n://us-east-1.elasticmapreduce/libs/hive/']
-
-
-class InstallHiveStep(HiveBase):
- """
- Install Hive on EMR step
- """
- InstallHiveName = 'Install Hive'
-
- def __init__(self, hive_versions='latest', hive_site=None):
- step_args = []
- step_args.extend(self.BaseArgs)
- step_args.extend(['--install-hive'])
- step_args.extend(['--hive-versions', hive_versions])
- if hive_site is not None:
- step_args.extend(['--hive-site=%s' % hive_site])
- super(InstallHiveStep, self).__init__(self.InstallHiveName,
- step_args=step_args)
-
-
-class HiveStep(HiveBase):
- """
- Hive script step
- """
-
- def __init__(self, name, hive_file, hive_versions='latest',
- hive_args=None):
- step_args = []
- step_args.extend(self.BaseArgs)
- step_args.extend(['--hive-versions', hive_versions])
- step_args.extend(['--run-hive-script', '--args', '-f', hive_file])
- if hive_args is not None:
- step_args.extend(hive_args)
- super(HiveStep, self).__init__(name, step_args=step_args)

Powered by Google App Engine
This is Rietveld 408576698