OLD | NEW |
1 #!/usr/bin/python | 1 #!/usr/bin/python |
2 | 2 |
3 """ | 3 """ |
4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
5 | 5 |
6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
7 found in the LICENSE file. | 7 found in the LICENSE file. |
8 | 8 |
9 HTTP server for our HTML rebaseline viewer. | 9 HTTP server for our HTML rebaseline viewer. |
10 """ | 10 """ |
(...skipping 200 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
211 """ | 211 """ |
212 Args: | 212 Args: |
213 actuals_dir: directory under which we will check out the latest actual | 213 actuals_dir: directory under which we will check out the latest actual |
214 GM results | 214 GM results |
215 json_filename: basename of the JSON summary file to load for each builder | 215 json_filename: basename of the JSON summary file to load for each builder |
216 gm_summaries_bucket: Google Storage bucket to download json_filename | 216 gm_summaries_bucket: Google Storage bucket to download json_filename |
217 files from; if None or '', don't fetch new actual-results files | 217 files from; if None or '', don't fetch new actual-results files |
218 at all, just compare to whatever files are already in actuals_dir | 218 at all, just compare to whatever files are already in actuals_dir |
219 port: which TCP port to listen on for HTTP requests | 219 port: which TCP port to listen on for HTTP requests |
220 export: whether to allow HTTP clients on other hosts to access this server | 220 export: whether to allow HTTP clients on other hosts to access this server |
221 editable: whether HTTP clients are allowed to submit new baselines | 221 editable: whether HTTP clients are allowed to submit new GM baselines |
| 222 (SKP baseline modifications are performed using an entirely different |
| 223 mechanism, not affected by this parameter) |
222 reload_seconds: polling interval with which to check for new results; | 224 reload_seconds: polling interval with which to check for new results; |
223 if 0, don't check for new results at all | 225 if 0, don't check for new results at all |
224 config_pairs: List of (string, string) tuples; for each tuple, compare | 226 config_pairs: List of (string, string) tuples; for each tuple, compare |
225 actual results of these two configs. If None or empty, | 227 actual results of these two configs. If None or empty, |
226 don't compare configs at all. | 228 don't compare configs at all. |
227 builder_regex_list: List of regular expressions specifying which builders | 229 builder_regex_list: List of regular expressions specifying which builders |
228 we will process. If None, process all builders. | 230 we will process. If None, process all builders. |
229 boto_file_path: Path to .boto file giving us credentials to access | 231 boto_file_path: Path to .boto file giving us credentials to access |
230 Google Storage buckets; if None, we will only be able to access | 232 Google Storage buckets; if None, we will only be able to access |
231 public GS buckets. | 233 public GS buckets. |
(...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
286 return self._gs | 288 return self._gs |
287 | 289 |
288 @property | 290 @property |
289 def is_exported(self): | 291 def is_exported(self): |
290 """ Returns true iff HTTP clients on other hosts are allowed to access | 292 """ Returns true iff HTTP clients on other hosts are allowed to access |
291 this server. """ | 293 this server. """ |
292 return self._export | 294 return self._export |
293 | 295 |
294 @property | 296 @property |
295 def is_editable(self): | 297 def is_editable(self): |
296 """ Returns true iff HTTP clients are allowed to submit new baselines. """ | 298 """ True iff HTTP clients are allowed to submit new GM baselines. """ |
297 return self._editable | 299 return self._editable |
298 | 300 |
299 @property | 301 @property |
300 def reload_seconds(self): | 302 def reload_seconds(self): |
301 """ Returns the result reload period in seconds, or 0 if we don't reload | 303 """ Returns the result reload period in seconds, or 0 if we don't reload |
302 results. """ | 304 results. """ |
303 return self._reload_seconds | 305 return self._reload_seconds |
304 | 306 |
305 def update_results(self, invalidate=False): | 307 def update_results(self, invalidate=False): |
306 """ Create or update self._results, based on the latest expectations and | 308 """ Create or update self._results, based on the latest expectations and |
(...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
517 results_mod.KEY__HEADER__SCHEMA_VERSION: ( | 519 results_mod.KEY__HEADER__SCHEMA_VERSION: ( |
518 results_mod.VALUE__HEADER__SCHEMA_VERSION), | 520 results_mod.VALUE__HEADER__SCHEMA_VERSION), |
519 results_mod.KEY__HEADER__IS_STILL_LOADING: True, | 521 results_mod.KEY__HEADER__IS_STILL_LOADING: True, |
520 results_mod.KEY__HEADER__TIME_UPDATED: now, | 522 results_mod.KEY__HEADER__TIME_UPDATED: now, |
521 results_mod.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( | 523 results_mod.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
522 now + RELOAD_INTERVAL_UNTIL_READY), | 524 now + RELOAD_INTERVAL_UNTIL_READY), |
523 }, | 525 }, |
524 } | 526 } |
525 self.send_json_dict(response_dict) | 527 self.send_json_dict(response_dict) |
526 | 528 |
| 529 def _get_live_results_or_prefetch(self, url_remainder, prefetch_only=False): |
| 530 """ Handle a GET request for live-generated image diff data. |
| 531 |
| 532 Args: |
| 533 url_remainder: string indicating which image diffs to generate |
| 534 prefetch_only: if True, the user isn't waiting around for results |
| 535 """ |
| 536 param_dict = urlparse.parse_qs(url_remainder) |
| 537 download_all_images = ( |
| 538 param_dict.get(LIVE_PARAM__DOWNLOAD_ONLY_DIFFERING, [''])[0].lower() |
| 539 not in ['1', 'true']) |
| 540 setA_section = self._validate_summary_section( |
| 541 param_dict.get(LIVE_PARAM__SET_A_SECTION, [None])[0]) |
| 542 setB_section = self._validate_summary_section( |
| 543 param_dict.get(LIVE_PARAM__SET_B_SECTION, [None])[0]) |
| 544 |
| 545 # Is the user in a position to submit new baselines? |
| 546 # |
| 547 # TODO(epoger): For now, we only allow rebaselining if the expectations |
| 548 # are on the left-hand side. In a coming CL, I would like to automatically |
| 549 # flip the two sides if a user requests setA=actuals, setB=expected. |
| 550 # Right now that is difficult to do, though, because the JSON generated by |
| 551 # RenderedPicturesComparisons doesn't include full setA/setB descriptions, |
| 552 # so the UI has to trust that whatever setA/setB were requested were also |
| 553 # returned. |
| 554 is_rebaselining = ((setA_section == gm_json.JSONKEY_EXPECTEDRESULTS) and |
| 555 (setB_section == gm_json.JSONKEY_ACTUALRESULTS)) |
| 556 |
| 557 results_obj = compare_rendered_pictures.RenderedPicturesComparisons( |
| 558 setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], |
| 559 setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], |
| 560 setA_section=setA_section, setB_section=setB_section, |
| 561 image_diff_db=_SERVER.image_diff_db, |
| 562 diff_base_url='/static/generated-images', |
| 563 gs=_SERVER.gs, truncate_results=_SERVER.truncate_results, |
| 564 prefetch_only=prefetch_only, download_all_images=download_all_images) |
| 565 |
| 566 if prefetch_only: |
| 567 self.send_response(200) |
| 568 else: |
| 569 self.send_json_dict(results_obj.get_packaged_results_of_type( |
| 570 results_mod.KEY__HEADER__RESULTS_ALL, is_editable=is_rebaselining)) |
| 571 |
527 def do_GET_live_results(self, url_remainder): | 572 def do_GET_live_results(self, url_remainder): |
528 """ Handle a GET request for live-generated image diff data. | 573 """ Handle a GET request for live-generated image diff data. |
529 | 574 |
530 Args: | 575 Args: |
531 url_remainder: string indicating which image diffs to generate | 576 url_remainder: string indicating which image diffs to generate |
532 """ | 577 """ |
533 logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) | 578 logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) |
534 param_dict = urlparse.parse_qs(url_remainder) | 579 self._get_live_results_or_prefetch( |
535 results_obj = self._call_compare_rendered_pictures( | 580 url_remainder=url_remainder, prefetch_only=False) |
536 param_dict=param_dict, prefetch_only=False) | |
537 self.send_json_dict(results_obj.get_packaged_results_of_type( | |
538 results_mod.KEY__HEADER__RESULTS_ALL)) | |
539 | 581 |
540 def do_GET_prefetch_results(self, url_remainder): | 582 def do_GET_prefetch_results(self, url_remainder): |
541 """ Prefetch image diff data for a future do_GET_live_results() call. | 583 """ Prefetch image diff data for a future do_GET_live_results() call. |
542 | 584 |
543 Args: | 585 Args: |
544 url_remainder: string indicating which image diffs to generate | 586 url_remainder: string indicating which image diffs to generate |
545 """ | 587 """ |
546 logging.debug('do_GET_prefetch_results: url_remainder="%s"' % url_remainder) | 588 logging.debug('do_GET_prefetch_results: url_remainder="%s"' % url_remainder) |
547 param_dict = urlparse.parse_qs(url_remainder) | 589 self._get_live_results_or_prefetch( |
548 self._call_compare_rendered_pictures( | 590 url_remainder=url_remainder, prefetch_only=True) |
549 param_dict=param_dict, prefetch_only=True) | |
550 self.send_response(200) | |
551 | 591 |
552 def do_GET_static(self, path): | 592 def do_GET_static(self, path): |
553 """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . | 593 """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . |
554 Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a | 594 Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a |
555 filesystem sibling of this script. | 595 filesystem sibling of this script. |
556 | 596 |
557 Args: | 597 Args: |
558 path: path to file (within STATIC_CONTENTS_SUBDIR) to retrieve | 598 path: path to file (within STATIC_CONTENTS_SUBDIR) to retrieve |
559 """ | 599 """ |
560 # Strip arguments ('?resultsToLoad=all') from the path | 600 # Strip arguments ('?resultsToLoad=all') from the path |
(...skipping 126 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
687 mimetype. | 727 mimetype. |
688 | 728 |
689 Args: | 729 Args: |
690 json_dict: dictionary to send | 730 json_dict: dictionary to send |
691 """ | 731 """ |
692 self.send_response(200) | 732 self.send_response(200) |
693 self.send_header('Content-type', 'application/json') | 733 self.send_header('Content-type', 'application/json') |
694 self.end_headers() | 734 self.end_headers() |
695 json.dump(json_dict, self.wfile) | 735 json.dump(json_dict, self.wfile) |
696 | 736 |
697 def _call_compare_rendered_pictures(self, param_dict, prefetch_only): | |
698 """Instantiates RenderedPicturesComparisons object to serve a GET request. | |
699 | |
700 Args: | |
701 param_dict: dictionary of URL parameters | |
702 prefetch_only: parameter to pass into RenderedPicturesComparisons | |
703 constructor | |
704 | |
705 Returns: a reference to the new RenderedPicturesComparisons object. | |
706 """ | |
707 download_all_images = ( | |
708 param_dict.get(LIVE_PARAM__DOWNLOAD_ONLY_DIFFERING, [''])[0].lower() | |
709 not in ['1', 'true']) | |
710 setA_section = self._validate_summary_section( | |
711 param_dict.get(LIVE_PARAM__SET_A_SECTION, [None])[0]) | |
712 setB_section = self._validate_summary_section( | |
713 param_dict.get(LIVE_PARAM__SET_B_SECTION, [None])[0]) | |
714 return compare_rendered_pictures.RenderedPicturesComparisons( | |
715 setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], | |
716 setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], | |
717 setA_section=setA_section, setB_section=setB_section, | |
718 image_diff_db=_SERVER.image_diff_db, | |
719 diff_base_url='/static/generated-images', | |
720 gs=_SERVER.gs, truncate_results=_SERVER.truncate_results, | |
721 prefetch_only=prefetch_only, download_all_images=download_all_images) | |
722 | |
723 def _validate_summary_section(self, section_name): | 737 def _validate_summary_section(self, section_name): |
724 """Validates the section we have been requested to read within JSON summary. | 738 """Validates the section we have been requested to read within JSON summary. |
725 | 739 |
726 Args: | 740 Args: |
727 section_name: which section of the JSON summary file has been requested | 741 section_name: which section of the JSON summary file has been requested |
728 | 742 |
729 Returns: the validated section name | 743 Returns: the validated section name |
730 | 744 |
731 Raises: Exception if an invalid section_name was requested. | 745 Raises: Exception if an invalid section_name was requested. |
732 """ | 746 """ |
(...skipping 27 matching lines...) Expand all Loading... |
760 parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', | 774 parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', |
761 help=('Only process builders matching these regular ' | 775 help=('Only process builders matching these regular ' |
762 'expressions. If unspecified, process all ' | 776 'expressions. If unspecified, process all ' |
763 'builders.')) | 777 'builders.')) |
764 parser.add_argument('--compare-configs', action='store_true', | 778 parser.add_argument('--compare-configs', action='store_true', |
765 help=('In addition to generating differences between ' | 779 help=('In addition to generating differences between ' |
766 'expectations and actuals, also generate ' | 780 'expectations and actuals, also generate ' |
767 'differences between these config pairs: ' | 781 'differences between these config pairs: ' |
768 + str(CONFIG_PAIRS_TO_COMPARE))) | 782 + str(CONFIG_PAIRS_TO_COMPARE))) |
769 parser.add_argument('--editable', action='store_true', | 783 parser.add_argument('--editable', action='store_true', |
770 help=('Allow HTTP clients to submit new baselines.')) | 784 help=('Allow HTTP clients to submit new GM baselines.')) |
771 parser.add_argument('--export', action='store_true', | 785 parser.add_argument('--export', action='store_true', |
772 help=('Instead of only allowing access from HTTP clients ' | 786 help=('Instead of only allowing access from HTTP clients ' |
773 'on localhost, allow HTTP clients on other hosts ' | 787 'on localhost, allow HTTP clients on other hosts ' |
774 'to access this server. WARNING: doing so will ' | 788 'to access this server. WARNING: doing so will ' |
775 'allow users on other hosts to modify your ' | 789 'allow users on other hosts to modify your ' |
776 'GM expectations, if combined with --editable.')) | 790 'GM expectations, if combined with --editable.')) |
777 parser.add_argument('--gm-summaries-bucket', | 791 parser.add_argument('--gm-summaries-bucket', |
778 help=('Google Cloud Storage bucket to download ' | 792 help=('Google Cloud Storage bucket to download ' |
779 'JSON_FILENAME files from. ' | 793 'JSON_FILENAME files from. ' |
780 'Defaults to %(default)s ; if set to ' | 794 'Defaults to %(default)s ; if set to ' |
(...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
819 reload_seconds=args.reload, config_pairs=config_pairs, | 833 reload_seconds=args.reload, config_pairs=config_pairs, |
820 builder_regex_list=args.builders, boto_file_path=args.boto, | 834 builder_regex_list=args.builders, boto_file_path=args.boto, |
821 imagediffdb_threads=args.threads) | 835 imagediffdb_threads=args.threads) |
822 if args.truncate: | 836 if args.truncate: |
823 _SERVER.truncate_results = True | 837 _SERVER.truncate_results = True |
824 _SERVER.run() | 838 _SERVER.run() |
825 | 839 |
826 | 840 |
827 if __name__ == '__main__': | 841 if __name__ == '__main__': |
828 main() | 842 main() |
OLD | NEW |