Chromium Code Reviews Chromium Code Reviews
Issue 1505303003:
[perf-insights] Trace corpus cleanup endpoint (Closed)
Base URL: https://github.com/catapult-project/catapult.git@master| Index: perf_insights/perf_insights/endpoints/corpus_cleanup.py |
| diff --git a/perf_insights/perf_insights/endpoints/corpus_cleanup.py b/perf_insights/perf_insights/endpoints/corpus_cleanup.py |
| new file mode 100644 |
| index 0000000000000000000000000000000000000000..be6f0784a3060cda2cdb59d386911a33f573131a |
| --- /dev/null |
| +++ b/perf_insights/perf_insights/endpoints/corpus_cleanup.py |
| @@ -0,0 +1,64 @@ |
| +# Copyright 2015 The Chromium Authors. All rights reserved. |
| +# Use of this source code is governed by a BSD-style license that can be |
| +# found in the LICENSE file. |
| + |
| +import datetime |
| +import json |
|
shatch
2015/12/09 15:00:46
nit: don't think this and datastore_errors are use
oystein (OOO til 10th of July)
2015/12/09 19:20:35
Done.
|
| +import logging |
| +import time |
| +import webapp2 |
| + |
| +from perf_insights.trace_info import TraceInfo |
|
shatch
2015/12/09 15:00:46
nit: order
oystein (OOO til 10th of July)
2015/12/09 19:20:34
Done.
|
| +from perf_insights import cloud_config |
| + |
| +import third_party.cloudstorage as gcs |
| + |
| +from google.appengine.api import datastore_errors |
| + |
| +default_retry_params = gcs.RetryParams(initial_delay=0.2, |
|
shatch
2015/12/09 15:00:46
nit: Could probably just import the cloud_helper a
oystein (OOO til 10th of July)
2015/12/09 19:20:34
Done.
|
| + max_delay=5.0, |
| + backoff_factor=2, |
| + max_retry_period=15) |
| +gcs.set_default_retry_params(default_retry_params) |
| + |
| +BATCH_SIZE=100 |
| +MAX_DAYS=30 |
| + |
| +class CorpusCleanupPage(webapp2.RequestHandler): |
| + |
| + def _delete_traces(self): |
| + trace_bucket = cloud_config.Get().trace_upload_bucket |
| + deleted_traces = 0; |
| + |
| + oldest_time = datetime.datetime.now() - datetime.timedelta(days=MAX_DAYS) |
| + q = TraceInfo.query(TraceInfo.date < oldest_time) |
| + |
| + for key in q.fetch(BATCH_SIZE, keys_only=True): |
| + gcs_path = '/%s/%s.gz' % (trace_bucket, key.id()) |
| + try: |
| + gcs.delete(gcs_path, retry_params=default_retry_params) |
| + except gcs.NotFoundError: |
| + pass |
| + |
| + key.delete() |
| + deleted_traces += 1 |
| + |
| + return deleted_traces |
| + |
| + def get(self): |
| + self.response.out.write('<html><body>') |
| + |
| + while True: |
| + deleted_traces = self._delete_traces(); |
| + self.response.out.write("<br><div><bold>Traces Cleaned:</bold> %s</div>" |
| + % deleted_traces) |
| + |
| + logging.info('Daily cleanup deleted %s traces.' % deleted_traces) |
| + |
| + if deleted_traces < BATCH_SIZE: |
| + break |
| + |
| + self.response.out.write('</body></html>') |
| + |
| + |
| +app = webapp2.WSGIApplication([('/corpus_cleanup', CorpusCleanupPage)]) |
