| OLD | NEW |
| 1 #!/usr/bin/python | 1 #!/usr/bin/python |
| 2 | 2 |
| 3 """ | 3 """ |
| 4 Copyright 2013 Google Inc. | 4 Copyright 2013 Google Inc. |
| 5 | 5 |
| 6 Use of this source code is governed by a BSD-style license that can be | 6 Use of this source code is governed by a BSD-style license that can be |
| 7 found in the LICENSE file. | 7 found in the LICENSE file. |
| 8 | 8 |
| 9 HTTP server for our HTML rebaseline viewer. | 9 HTTP server for our HTML rebaseline viewer. |
| 10 """ | 10 """ |
| (...skipping 200 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 211 """ | 211 """ |
| 212 Args: | 212 Args: |
| 213 actuals_dir: directory under which we will check out the latest actual | 213 actuals_dir: directory under which we will check out the latest actual |
| 214 GM results | 214 GM results |
| 215 json_filename: basename of the JSON summary file to load for each builder | 215 json_filename: basename of the JSON summary file to load for each builder |
| 216 gm_summaries_bucket: Google Storage bucket to download json_filename | 216 gm_summaries_bucket: Google Storage bucket to download json_filename |
| 217 files from; if None or '', don't fetch new actual-results files | 217 files from; if None or '', don't fetch new actual-results files |
| 218 at all, just compare to whatever files are already in actuals_dir | 218 at all, just compare to whatever files are already in actuals_dir |
| 219 port: which TCP port to listen on for HTTP requests | 219 port: which TCP port to listen on for HTTP requests |
| 220 export: whether to allow HTTP clients on other hosts to access this server | 220 export: whether to allow HTTP clients on other hosts to access this server |
| 221 editable: whether HTTP clients are allowed to submit new baselines | 221 editable: whether HTTP clients are allowed to submit new GM baselines |
| 222 (SKP baseline modifications are performed using an entirely different |
| 223 mechanism, not affected by this parameter) |
| 222 reload_seconds: polling interval with which to check for new results; | 224 reload_seconds: polling interval with which to check for new results; |
| 223 if 0, don't check for new results at all | 225 if 0, don't check for new results at all |
| 224 config_pairs: List of (string, string) tuples; for each tuple, compare | 226 config_pairs: List of (string, string) tuples; for each tuple, compare |
| 225 actual results of these two configs. If None or empty, | 227 actual results of these two configs. If None or empty, |
| 226 don't compare configs at all. | 228 don't compare configs at all. |
| 227 builder_regex_list: List of regular expressions specifying which builders | 229 builder_regex_list: List of regular expressions specifying which builders |
| 228 we will process. If None, process all builders. | 230 we will process. If None, process all builders. |
| 229 boto_file_path: Path to .boto file giving us credentials to access | 231 boto_file_path: Path to .boto file giving us credentials to access |
| 230 Google Storage buckets; if None, we will only be able to access | 232 Google Storage buckets; if None, we will only be able to access |
| 231 public GS buckets. | 233 public GS buckets. |
| (...skipping 54 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 286 return self._gs | 288 return self._gs |
| 287 | 289 |
| 288 @property | 290 @property |
| 289 def is_exported(self): | 291 def is_exported(self): |
| 290 """ Returns true iff HTTP clients on other hosts are allowed to access | 292 """ Returns true iff HTTP clients on other hosts are allowed to access |
| 291 this server. """ | 293 this server. """ |
| 292 return self._export | 294 return self._export |
| 293 | 295 |
| 294 @property | 296 @property |
| 295 def is_editable(self): | 297 def is_editable(self): |
| 296 """ Returns true iff HTTP clients are allowed to submit new baselines. """ | 298 """ True iff HTTP clients are allowed to submit new GM baselines. """ |
| 297 return self._editable | 299 return self._editable |
| 298 | 300 |
| 299 @property | 301 @property |
| 300 def reload_seconds(self): | 302 def reload_seconds(self): |
| 301 """ Returns the result reload period in seconds, or 0 if we don't reload | 303 """ Returns the result reload period in seconds, or 0 if we don't reload |
| 302 results. """ | 304 results. """ |
| 303 return self._reload_seconds | 305 return self._reload_seconds |
| 304 | 306 |
| 305 def update_results(self, invalidate=False): | 307 def update_results(self, invalidate=False): |
| 306 """ Create or update self._results, based on the latest expectations and | 308 """ Create or update self._results, based on the latest expectations and |
| (...skipping 210 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 517 results_mod.KEY__HEADER__SCHEMA_VERSION: ( | 519 results_mod.KEY__HEADER__SCHEMA_VERSION: ( |
| 518 results_mod.VALUE__HEADER__SCHEMA_VERSION), | 520 results_mod.VALUE__HEADER__SCHEMA_VERSION), |
| 519 results_mod.KEY__HEADER__IS_STILL_LOADING: True, | 521 results_mod.KEY__HEADER__IS_STILL_LOADING: True, |
| 520 results_mod.KEY__HEADER__TIME_UPDATED: now, | 522 results_mod.KEY__HEADER__TIME_UPDATED: now, |
| 521 results_mod.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( | 523 results_mod.KEY__HEADER__TIME_NEXT_UPDATE_AVAILABLE: ( |
| 522 now + RELOAD_INTERVAL_UNTIL_READY), | 524 now + RELOAD_INTERVAL_UNTIL_READY), |
| 523 }, | 525 }, |
| 524 } | 526 } |
| 525 self.send_json_dict(response_dict) | 527 self.send_json_dict(response_dict) |
| 526 | 528 |
| 529 def _get_live_results_or_prefetch(self, url_remainder, prefetch_only=False): |
| 530 """ Handle a GET request for live-generated image diff data. |
| 531 |
| 532 Args: |
| 533 url_remainder: string indicating which image diffs to generate |
| 534 prefetch_only: if True, the user isn't waiting around for results |
| 535 """ |
| 536 param_dict = urlparse.parse_qs(url_remainder) |
| 537 download_all_images = ( |
| 538 param_dict.get(LIVE_PARAM__DOWNLOAD_ONLY_DIFFERING, [''])[0].lower() |
| 539 not in ['1', 'true']) |
| 540 setA_section = self._validate_summary_section( |
| 541 param_dict.get(LIVE_PARAM__SET_A_SECTION, [None])[0]) |
| 542 setB_section = self._validate_summary_section( |
| 543 param_dict.get(LIVE_PARAM__SET_B_SECTION, [None])[0]) |
| 544 |
| 545 # Is the user in a position to submit new baselines? |
| 546 # |
| 547 # TODO(epoger): For now, we only allow rebaselining if the expectations |
| 548 # are on the left-hand side. In a coming CL, I would like to automatically |
| 549 # flip the two sides if a user requests setA=actuals, setB=expected. |
| 550 # Right now that is difficult to do, though, because the JSON generated by |
| 551 # RenderedPicturesComparisons doesn't include full setA/setB descriptions, |
| 552 # so the UI has to trust that whatever setA/setB were requested were also |
| 553 # returned. |
| 554 is_rebaselining = ((setA_section == gm_json.JSONKEY_EXPECTEDRESULTS) and |
| 555 (setB_section == gm_json.JSONKEY_ACTUALRESULTS)) |
| 556 |
| 557 results_obj = compare_rendered_pictures.RenderedPicturesComparisons( |
| 558 setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], |
| 559 setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], |
| 560 setA_section=setA_section, setB_section=setB_section, |
| 561 image_diff_db=_SERVER.image_diff_db, |
| 562 diff_base_url='/static/generated-images', |
| 563 gs=_SERVER.gs, truncate_results=_SERVER.truncate_results, |
| 564 prefetch_only=prefetch_only, download_all_images=download_all_images) |
| 565 |
| 566 if prefetch_only: |
| 567 self.send_response(200) |
| 568 else: |
| 569 self.send_json_dict(results_obj.get_packaged_results_of_type( |
| 570 results_mod.KEY__HEADER__RESULTS_ALL, is_editable=is_rebaselining)) |
| 571 |
| 527 def do_GET_live_results(self, url_remainder): | 572 def do_GET_live_results(self, url_remainder): |
| 528 """ Handle a GET request for live-generated image diff data. | 573 """ Handle a GET request for live-generated image diff data. |
| 529 | 574 |
| 530 Args: | 575 Args: |
| 531 url_remainder: string indicating which image diffs to generate | 576 url_remainder: string indicating which image diffs to generate |
| 532 """ | 577 """ |
| 533 logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) | 578 logging.debug('do_GET_live_results: url_remainder="%s"' % url_remainder) |
| 534 param_dict = urlparse.parse_qs(url_remainder) | 579 self._get_live_results_or_prefetch( |
| 535 results_obj = self._call_compare_rendered_pictures( | 580 url_remainder=url_remainder, prefetch_only=False) |
| 536 param_dict=param_dict, prefetch_only=False) | |
| 537 self.send_json_dict(results_obj.get_packaged_results_of_type( | |
| 538 results_mod.KEY__HEADER__RESULTS_ALL)) | |
| 539 | 581 |
| 540 def do_GET_prefetch_results(self, url_remainder): | 582 def do_GET_prefetch_results(self, url_remainder): |
| 541 """ Prefetch image diff data for a future do_GET_live_results() call. | 583 """ Prefetch image diff data for a future do_GET_live_results() call. |
| 542 | 584 |
| 543 Args: | 585 Args: |
| 544 url_remainder: string indicating which image diffs to generate | 586 url_remainder: string indicating which image diffs to generate |
| 545 """ | 587 """ |
| 546 logging.debug('do_GET_prefetch_results: url_remainder="%s"' % url_remainder) | 588 logging.debug('do_GET_prefetch_results: url_remainder="%s"' % url_remainder) |
| 547 param_dict = urlparse.parse_qs(url_remainder) | 589 self._get_live_results_or_prefetch( |
| 548 self._call_compare_rendered_pictures( | 590 url_remainder=url_remainder, prefetch_only=True) |
| 549 param_dict=param_dict, prefetch_only=True) | |
| 550 self.send_response(200) | |
| 551 | 591 |
| 552 def do_GET_static(self, path): | 592 def do_GET_static(self, path): |
| 553 """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . | 593 """ Handle a GET request for a file under STATIC_CONTENTS_SUBDIR . |
| 554 Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a | 594 Only allow serving of files within STATIC_CONTENTS_SUBDIR that is a |
| 555 filesystem sibling of this script. | 595 filesystem sibling of this script. |
| 556 | 596 |
| 557 Args: | 597 Args: |
| 558 path: path to file (within STATIC_CONTENTS_SUBDIR) to retrieve | 598 path: path to file (within STATIC_CONTENTS_SUBDIR) to retrieve |
| 559 """ | 599 """ |
| 560 # Strip arguments ('?resultsToLoad=all') from the path | 600 # Strip arguments ('?resultsToLoad=all') from the path |
| (...skipping 126 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 687 mimetype. | 727 mimetype. |
| 688 | 728 |
| 689 Args: | 729 Args: |
| 690 json_dict: dictionary to send | 730 json_dict: dictionary to send |
| 691 """ | 731 """ |
| 692 self.send_response(200) | 732 self.send_response(200) |
| 693 self.send_header('Content-type', 'application/json') | 733 self.send_header('Content-type', 'application/json') |
| 694 self.end_headers() | 734 self.end_headers() |
| 695 json.dump(json_dict, self.wfile) | 735 json.dump(json_dict, self.wfile) |
| 696 | 736 |
| 697 def _call_compare_rendered_pictures(self, param_dict, prefetch_only): | |
| 698 """Instantiates RenderedPicturesComparisons object to serve a GET request. | |
| 699 | |
| 700 Args: | |
| 701 param_dict: dictionary of URL parameters | |
| 702 prefetch_only: parameter to pass into RenderedPicturesComparisons | |
| 703 constructor | |
| 704 | |
| 705 Returns: a reference to the new RenderedPicturesComparisons object. | |
| 706 """ | |
| 707 download_all_images = ( | |
| 708 param_dict.get(LIVE_PARAM__DOWNLOAD_ONLY_DIFFERING, [''])[0].lower() | |
| 709 not in ['1', 'true']) | |
| 710 setA_section = self._validate_summary_section( | |
| 711 param_dict.get(LIVE_PARAM__SET_A_SECTION, [None])[0]) | |
| 712 setB_section = self._validate_summary_section( | |
| 713 param_dict.get(LIVE_PARAM__SET_B_SECTION, [None])[0]) | |
| 714 return compare_rendered_pictures.RenderedPicturesComparisons( | |
| 715 setA_dirs=param_dict[LIVE_PARAM__SET_A_DIR], | |
| 716 setB_dirs=param_dict[LIVE_PARAM__SET_B_DIR], | |
| 717 setA_section=setA_section, setB_section=setB_section, | |
| 718 image_diff_db=_SERVER.image_diff_db, | |
| 719 diff_base_url='/static/generated-images', | |
| 720 gs=_SERVER.gs, truncate_results=_SERVER.truncate_results, | |
| 721 prefetch_only=prefetch_only, download_all_images=download_all_images) | |
| 722 | |
| 723 def _validate_summary_section(self, section_name): | 737 def _validate_summary_section(self, section_name): |
| 724 """Validates the section we have been requested to read within JSON summary. | 738 """Validates the section we have been requested to read within JSON summary. |
| 725 | 739 |
| 726 Args: | 740 Args: |
| 727 section_name: which section of the JSON summary file has been requested | 741 section_name: which section of the JSON summary file has been requested |
| 728 | 742 |
| 729 Returns: the validated section name | 743 Returns: the validated section name |
| 730 | 744 |
| 731 Raises: Exception if an invalid section_name was requested. | 745 Raises: Exception if an invalid section_name was requested. |
| 732 """ | 746 """ |
| (...skipping 27 matching lines...) Expand all Loading... |
| 760 parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', | 774 parser.add_argument('--builders', metavar='BUILDER_REGEX', nargs='+', |
| 761 help=('Only process builders matching these regular ' | 775 help=('Only process builders matching these regular ' |
| 762 'expressions. If unspecified, process all ' | 776 'expressions. If unspecified, process all ' |
| 763 'builders.')) | 777 'builders.')) |
| 764 parser.add_argument('--compare-configs', action='store_true', | 778 parser.add_argument('--compare-configs', action='store_true', |
| 765 help=('In addition to generating differences between ' | 779 help=('In addition to generating differences between ' |
| 766 'expectations and actuals, also generate ' | 780 'expectations and actuals, also generate ' |
| 767 'differences between these config pairs: ' | 781 'differences between these config pairs: ' |
| 768 + str(CONFIG_PAIRS_TO_COMPARE))) | 782 + str(CONFIG_PAIRS_TO_COMPARE))) |
| 769 parser.add_argument('--editable', action='store_true', | 783 parser.add_argument('--editable', action='store_true', |
| 770 help=('Allow HTTP clients to submit new baselines.')) | 784 help=('Allow HTTP clients to submit new GM baselines.')) |
| 771 parser.add_argument('--export', action='store_true', | 785 parser.add_argument('--export', action='store_true', |
| 772 help=('Instead of only allowing access from HTTP clients ' | 786 help=('Instead of only allowing access from HTTP clients ' |
| 773 'on localhost, allow HTTP clients on other hosts ' | 787 'on localhost, allow HTTP clients on other hosts ' |
| 774 'to access this server. WARNING: doing so will ' | 788 'to access this server. WARNING: doing so will ' |
| 775 'allow users on other hosts to modify your ' | 789 'allow users on other hosts to modify your ' |
| 776 'GM expectations, if combined with --editable.')) | 790 'GM expectations, if combined with --editable.')) |
| 777 parser.add_argument('--gm-summaries-bucket', | 791 parser.add_argument('--gm-summaries-bucket', |
| 778 help=('Google Cloud Storage bucket to download ' | 792 help=('Google Cloud Storage bucket to download ' |
| 779 'JSON_FILENAME files from. ' | 793 'JSON_FILENAME files from. ' |
| 780 'Defaults to %(default)s ; if set to ' | 794 'Defaults to %(default)s ; if set to ' |
| (...skipping 38 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 819 reload_seconds=args.reload, config_pairs=config_pairs, | 833 reload_seconds=args.reload, config_pairs=config_pairs, |
| 820 builder_regex_list=args.builders, boto_file_path=args.boto, | 834 builder_regex_list=args.builders, boto_file_path=args.boto, |
| 821 imagediffdb_threads=args.threads) | 835 imagediffdb_threads=args.threads) |
| 822 if args.truncate: | 836 if args.truncate: |
| 823 _SERVER.truncate_results = True | 837 _SERVER.truncate_results = True |
| 824 _SERVER.run() | 838 _SERVER.run() |
| 825 | 839 |
| 826 | 840 |
| 827 if __name__ == '__main__': | 841 if __name__ == '__main__': |
| 828 main() | 842 main() |
| OLD | NEW |