OLD | NEW |
---|---|
1 # Copyright 2015 Google Inc. All rights reserved. | 1 # Copyright 2015 Google Inc. All rights reserved. |
2 # | 2 # |
3 # Licensed under the Apache License, Version 2.0 (the "License"); | 3 # Licensed under the Apache License, Version 2.0 (the "License"); |
4 # you may not use this file except in compliance with the License. | 4 # you may not use this file except in compliance with the License. |
5 # You may obtain a copy of the License at | 5 # You may obtain a copy of the License at |
6 # | 6 # |
7 # http://www.apache.org/licenses/LICENSE-2.0 | 7 # http://www.apache.org/licenses/LICENSE-2.0 |
8 # | 8 # |
9 # Unless required by applicable law or agreed to in writing, software | 9 # Unless required by applicable law or agreed to in writing, software |
10 # distributed under the License is distributed on an "AS IS" BASIS, | 10 # distributed under the License is distributed on an "AS IS" BASIS, |
(...skipping 47 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... | |
58 while len(pool) < len(items) and len(pool) < 64: | 58 while len(pool) < len(items) and len(pool) < 64: |
59 t = threading.Thread( | 59 t = threading.Thread( |
60 target=_run, name='parallel%d' % len(pool), args=(queue_in, queue_out)) | 60 target=_run, name='parallel%d' % len(pool), args=(queue_in, queue_out)) |
61 t.daemon = True | 61 t.daemon = True |
62 t.start() | 62 t.start() |
63 pool.append(t) | 63 pool.append(t) |
64 | 64 |
65 for index, item in enumerate(items): | 65 for index, item in enumerate(items): |
66 queue_in.put((index, fn, item)) | 66 queue_in.put((index, fn, item)) |
67 out = [None] * len(items) | 67 out = [None] * len(items) |
68 e = None | |
68 for _ in xrange(len(items)): | 69 for _ in xrange(len(items)): |
69 index, result = queue_out.get() | 70 index, result = queue_out.get() |
70 if index < 0: | 71 if index < 0: |
71 # This is an exception. | 72 # This is an exception. |
72 raise result[0], result[1], result[2] | 73 e = result |
ghost stip (do not use)
2015/11/03 18:50:47
and I guess we'll just clobber repeated errors
M-A Ruel
2015/11/04 18:39:10
yes
| |
73 out[index] = result | 74 else: |
75 out[index] = result | |
76 if e: | |
77 raise e[0], e[1], e[2] | |
74 return out | 78 return out |
75 | 79 |
76 | 80 |
77 def _run(queue_in, queue_out): | 81 def _run(queue_in, queue_out): |
78 while True: | 82 while True: |
79 index, fn, item = queue_in.get() | 83 index, fn, item = queue_in.get() |
80 try: | 84 try: |
81 result = fn(item) | 85 result = fn(item) |
82 except: # pylint: disable=bare-except | 86 except: # pylint: disable=bare-except |
83 # Starts at -1 otherwise -0 == 0. | 87 # Starts at -1 otherwise -0 == 0. |
84 index = -index - 1 | 88 index = -index - 1 |
85 result = sys.exc_info() | 89 result = sys.exc_info() |
86 finally: | 90 finally: |
87 queue_out.put((index, result)) | 91 queue_out.put((index, result)) |
OLD | NEW |