Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(203)

Unified Diff: tools/telemetry/telemetry/wpr/archive_info.py

Issue 1647513002: Delete tools/telemetry. (Closed) Base URL: https://chromium.googlesource.com/chromium/src.git@master
Patch Set: Created 4 years, 11 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View side-by-side diff with in-line comments
Download patch
« no previous file with comments | « tools/telemetry/telemetry/wpr/__init__.py ('k') | tools/telemetry/telemetry/wpr/archive_info_unittest.py » ('j') | no next file with comments »
Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
Index: tools/telemetry/telemetry/wpr/archive_info.py
diff --git a/tools/telemetry/telemetry/wpr/archive_info.py b/tools/telemetry/telemetry/wpr/archive_info.py
deleted file mode 100644
index 2e75829555135204867e8d8ef9a785505e6ca2e6..0000000000000000000000000000000000000000
--- a/tools/telemetry/telemetry/wpr/archive_info.py
+++ /dev/null
@@ -1,219 +0,0 @@
-# Copyright 2013 The Chromium Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-import json
-import logging
-import os
-import re
-import shutil
-import tempfile
-
-from catapult_base import cloud_storage # pylint: disable=import-error
-
-
-def AssertValidCloudStorageBucket(bucket):
- is_valid = bucket in (None,
- cloud_storage.PUBLIC_BUCKET,
- cloud_storage.PARTNER_BUCKET,
- cloud_storage.INTERNAL_BUCKET)
- if not is_valid:
- raise ValueError("Cloud storage privacy bucket %s is invalid" % bucket)
-
-
-class ArchiveError(Exception):
- pass
-
-
-class WprArchiveInfo(object):
- def __init__(self, file_path, data, bucket):
- AssertValidCloudStorageBucket(bucket)
- self._file_path = file_path
- self._base_dir = os.path.dirname(file_path)
- self._data = data
- self._bucket = bucket
-
- # Ensure directory exists.
- if not os.path.exists(self._base_dir):
- os.makedirs(self._base_dir)
-
- # Map from the relative path (as it appears in the metadata file) of the
- # .wpr file to a list of story names it supports.
- self._wpr_file_to_story_names = data['archives']
-
- # Map from the story name to a relative path (as it appears
- # in the metadata file) of the .wpr file.
- self._story_name_to_wpr_file = dict()
- # Find out the wpr file names for each story.
- for wpr_file in data['archives']:
- story_names = data['archives'][wpr_file]
- for story_name in story_names:
- self._story_name_to_wpr_file[story_name] = wpr_file
- self.temp_target_wpr_file_path = None
-
- @classmethod
- def FromFile(cls, file_path, bucket):
- if os.path.exists(file_path):
- with open(file_path, 'r') as f:
- data = json.load(f)
- return cls(file_path, data, bucket)
- return cls(file_path, {'archives': {}}, bucket)
-
- def DownloadArchivesIfNeeded(self):
- """Downloads archives iff the Archive has a bucket parameter and the user
- has permission to access the bucket.
-
- Raises cloud storage Permissions or Credentials error when there is no
- local copy of the archive and the user doesn't have permission to access
- the archive's bucket.
-
- Warns when a bucket is not specified or when the user doesn't have
- permission to access the archive's bucket but a local copy of the archive
- exists.
- """
- # Download all .wpr files.
- if not self._bucket:
- logging.warning('Story set in %s has no bucket specified, and '
- 'cannot be downloaded from cloud_storage.', )
- return
- assert 'archives' in self._data, 'Invalid data format in %s. \'archives\'' \
- ' field is needed' % self._file_path
- for archive_path in self._data['archives']:
- archive_path = self._WprFileNameToPath(archive_path)
- try:
- cloud_storage.GetIfChanged(archive_path, self._bucket)
- except (cloud_storage.CredentialsError, cloud_storage.PermissionError):
- if os.path.exists(archive_path):
- # If the archive exists, assume the user recorded their own and
- # simply warn.
- logging.warning('Need credentials to update WPR archive: %s',
- archive_path)
- else:
- logging.error("You either aren't authenticated or don't have "
- "permission to use the archives for this page set."
- "\nYou may need to run gsutil config."
- "\nYou can find instructions for gsutil config at: "
- "http://www.chromium.org/developers/telemetry/"
- "upload_to_cloud_storage")
- raise
-
- def WprFilePathForStory(self, story):
- if self.temp_target_wpr_file_path:
- return self.temp_target_wpr_file_path
- wpr_file = self._story_name_to_wpr_file.get(story.display_name, None)
- if wpr_file is None and hasattr(story, 'url'):
- # Some old pages always use the URL to identify a page rather than the
- # display_name, so try to look for that.
- wpr_file = self._story_name_to_wpr_file.get(story.url, None)
- if wpr_file:
- return self._WprFileNameToPath(wpr_file)
- return None
-
- def AddNewTemporaryRecording(self, temp_wpr_file_path=None):
- if temp_wpr_file_path is None:
- temp_wpr_file_handle, temp_wpr_file_path = tempfile.mkstemp()
- os.close(temp_wpr_file_handle)
- self.temp_target_wpr_file_path = temp_wpr_file_path
-
- def AddRecordedStories(self, stories, upload_to_cloud_storage=False):
- if not stories:
- os.remove(self.temp_target_wpr_file_path)
- return
-
- (target_wpr_file, target_wpr_file_path) = self._NextWprFileName()
- for story in stories:
- self._SetWprFileForStory(story.display_name, target_wpr_file)
- shutil.move(self.temp_target_wpr_file_path, target_wpr_file_path)
-
- # Update the hash file.
- target_wpr_file_hash = cloud_storage.CalculateHash(target_wpr_file_path)
- with open(target_wpr_file_path + '.sha1', 'wb') as f:
- f.write(target_wpr_file_hash)
- f.flush()
-
- self._WriteToFile()
- self._DeleteAbandonedWprFiles()
-
- # Upload to cloud storage
- if upload_to_cloud_storage:
- if not self._bucket:
- logging.warning('StorySet must have bucket specified to upload '
- 'stories to cloud storage.')
- return
- try:
- cloud_storage.Insert(self._bucket, target_wpr_file_hash,
- target_wpr_file_path)
- except cloud_storage.CloudStorageError, e:
- logging.warning('Failed to upload wpr file %s to cloud storage. '
- 'Error:%s' % target_wpr_file_path, e)
-
- def _DeleteAbandonedWprFiles(self):
- # Update the metadata so that the abandoned wpr files don't have
- # empty story name arrays.
- abandoned_wpr_files = self._AbandonedWprFiles()
- for wpr_file in abandoned_wpr_files:
- del self._wpr_file_to_story_names[wpr_file]
- # Don't fail if we're unable to delete some of the files.
- wpr_file_path = self._WprFileNameToPath(wpr_file)
- try:
- os.remove(wpr_file_path)
- except Exception:
- logging.warning('Failed to delete file: %s' % wpr_file_path)
-
- def _AbandonedWprFiles(self):
- abandoned_wpr_files = []
- for wpr_file, story_names in (
- self._wpr_file_to_story_names.iteritems()):
- if not story_names:
- abandoned_wpr_files.append(wpr_file)
- return abandoned_wpr_files
-
- def _WriteToFile(self):
- """Writes the metadata into the file passed as constructor parameter."""
- metadata = dict()
- metadata['description'] = (
- 'Describes the Web Page Replay archives for a story set. '
- 'Don\'t edit by hand! Use record_wpr for updating.')
- metadata['archives'] = self._wpr_file_to_story_names.copy()
- # Don't write data for abandoned archives.
- abandoned_wpr_files = self._AbandonedWprFiles()
- for wpr_file in abandoned_wpr_files:
- del metadata['archives'][wpr_file]
-
- with open(self._file_path, 'w') as f:
- json.dump(metadata, f, indent=4)
- f.flush()
-
- def _WprFileNameToPath(self, wpr_file):
- return os.path.abspath(os.path.join(self._base_dir, wpr_file))
-
- def _NextWprFileName(self):
- """Creates a new file name for a wpr archive file."""
- # The names are of the format "some_thing_number.wpr". Read the numbers.
- highest_number = -1
- base = None
- for wpr_file in self._wpr_file_to_story_names:
- match = re.match(r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+)\.wpr', wpr_file)
- if not match:
- raise Exception('Illegal wpr file name ' + wpr_file)
- highest_number = max(int(match.groupdict()['NUMBER']), highest_number)
- if base and match.groupdict()['BASE'] != base:
- raise Exception('Illegal wpr file name ' + wpr_file +
- ', doesn\'t begin with ' + base)
- base = match.groupdict()['BASE']
- if not base:
- # If we're creating a completely new info file, use the base name of the
- # story set file.
- base = os.path.splitext(os.path.basename(self._file_path))[0]
- new_filename = '%s_%03d.wpr' % (base, highest_number + 1)
- return new_filename, self._WprFileNameToPath(new_filename)
-
- def _SetWprFileForStory(self, story_name, wpr_file):
- """For modifying the metadata when we're going to record a new archive."""
- old_wpr_file = self._story_name_to_wpr_file.get(story_name, None)
- if old_wpr_file:
- self._wpr_file_to_story_names[old_wpr_file].remove(story_name)
- self._story_name_to_wpr_file[story_name] = wpr_file
- if wpr_file not in self._wpr_file_to_story_names:
- self._wpr_file_to_story_names[wpr_file] = []
- self._wpr_file_to_story_names[wpr_file].append(story_name)
« no previous file with comments | « tools/telemetry/telemetry/wpr/__init__.py ('k') | tools/telemetry/telemetry/wpr/archive_info_unittest.py » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698