OLD | NEW |
| (Empty) |
1 # -*- test-case-name: twisted.test.test_application,twisted.test.test_twistd -*- | |
2 # Copyright (c) 2001-2008 Twisted Matrix Laboratories. | |
3 # See LICENSE for details. | |
4 | |
5 import sys, os, pdb, getpass, traceback, signal, warnings | |
6 | |
7 from twisted.python import runtime, log, usage, failure, util, logfile | |
8 from twisted.persisted import sob | |
9 from twisted.application import service, reactors | |
10 from twisted.internet import defer | |
11 from twisted import copyright | |
12 | |
13 # Expose the new implementation of installReactor at the old location. | |
14 from twisted.application.reactors import installReactor | |
15 from twisted.application.reactors import NoSuchReactor | |
16 | |
17 | |
18 | |
19 class _BasicProfiler(object): | |
20 """ | |
21 @ivar saveStats: if C{True}, save the stats information instead of the | |
22 human readable format | |
23 @type saveStats: C{bool} | |
24 | |
25 @ivar profileOutput: the name of the file use to print profile data. | |
26 @type profileOutput: C{str} | |
27 """ | |
28 | |
29 def __init__(self, profileOutput, saveStats): | |
30 self.profileOutput = profileOutput | |
31 self.saveStats = saveStats | |
32 | |
33 | |
34 def _reportImportError(self, module, e): | |
35 """ | |
36 Helper method to report an import error with a profile module. This | |
37 has to be explicit because some of these modules are removed by | |
38 distributions due to them being non-free. | |
39 """ | |
40 s = "Failed to import module %s: %s" % (module, e) | |
41 s += """ | |
42 This is most likely caused by your operating system not including | |
43 the module due to it being non-free. Either do not use the option | |
44 --profile, or install the module; your operating system vendor | |
45 may provide it in a separate package. | |
46 """ | |
47 raise SystemExit(s) | |
48 | |
49 | |
50 | |
51 class ProfileRunner(_BasicProfiler): | |
52 """ | |
53 Runner for the standard profile module. | |
54 """ | |
55 | |
56 def run(self, reactor): | |
57 """ | |
58 Run reactor under the standard profiler. | |
59 """ | |
60 try: | |
61 import profile | |
62 except ImportError, e: | |
63 self._reportImportError("profile", e) | |
64 | |
65 p = profile.Profile() | |
66 p.runcall(reactor.run) | |
67 if self.saveStats: | |
68 p.dump_stats(self.profileOutput) | |
69 else: | |
70 tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'a') | |
71 try: | |
72 p.print_stats() | |
73 finally: | |
74 sys.stdout, tmp = tmp, sys.stdout | |
75 tmp.close() | |
76 | |
77 | |
78 | |
79 class HotshotRunner(_BasicProfiler): | |
80 """ | |
81 Runner for the hotshot profile module. | |
82 """ | |
83 | |
84 def run(self, reactor): | |
85 """ | |
86 Run reactor under the hotshot profiler. | |
87 """ | |
88 try: | |
89 import hotshot.stats | |
90 except (ImportError, SystemExit), e: | |
91 # Certain versions of Debian (and Debian derivatives) raise | |
92 # SystemExit when importing hotshot if the "non-free" profiler | |
93 # module is not installed. Someone eventually recognized this | |
94 # as a bug and changed the Debian packaged Python to raise | |
95 # ImportError instead. Handle both exception types here in | |
96 # order to support the versions of Debian which have this | |
97 # behavior. The bug report which prompted the introduction of | |
98 # this highly undesirable behavior should be available online at | |
99 # <http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=334067>. | |
100 # There seems to be no corresponding bug report which resulted | |
101 # in the behavior being removed. -exarkun | |
102 self._reportImportError("hotshot", e) | |
103 | |
104 # this writes stats straight out | |
105 p = hotshot.Profile(self.profileOutput) | |
106 p.runcall(reactor.run) | |
107 if self.saveStats: | |
108 # stats are automatically written to file, nothing to do | |
109 return | |
110 else: | |
111 s = hotshot.stats.load(self.profileOutput) | |
112 s.strip_dirs() | |
113 s.sort_stats(-1) | |
114 if getattr(s, 'stream', None) is not None: | |
115 # Python 2.5 and above supports a stream attribute | |
116 s.stream = open(self.profileOutput, 'w') | |
117 s.print_stats() | |
118 s.stream.close() | |
119 else: | |
120 # But we have to use a trick for Python < 2.5 | |
121 tmp, sys.stdout = sys.stdout, open(self.profileOutput, 'w') | |
122 try: | |
123 s.print_stats() | |
124 finally: | |
125 sys.stdout, tmp = tmp, sys.stdout | |
126 tmp.close() | |
127 | |
128 | |
129 | |
130 class CProfileRunner(_BasicProfiler): | |
131 """ | |
132 Runner for the cProfile module. | |
133 """ | |
134 | |
135 def run(self, reactor): | |
136 """ | |
137 Run reactor under the cProfile profiler. | |
138 """ | |
139 try: | |
140 import cProfile, pstats | |
141 except ImportError, e: | |
142 self._reportImportError("cProfile", e) | |
143 | |
144 p = cProfile.Profile() | |
145 p.runcall(reactor.run) | |
146 if self.saveStats: | |
147 p.dump_stats(self.profileOutput) | |
148 else: | |
149 stream = open(self.profileOutput, 'w') | |
150 s = pstats.Stats(p, stream=stream) | |
151 s.strip_dirs() | |
152 s.sort_stats(-1) | |
153 s.print_stats() | |
154 stream.close() | |
155 | |
156 | |
157 | |
158 class AppProfiler(object): | |
159 """ | |
160 Class which selects a specific profile runner based on configuration | |
161 options. | |
162 | |
163 @ivar profiler: the name of the selected profiler. | |
164 @type profiler: C{str} | |
165 """ | |
166 profilers = {"profile": ProfileRunner, "hotshot": HotshotRunner, | |
167 "cProfile": CProfileRunner} | |
168 | |
169 def __init__(self, options): | |
170 saveStats = options.get("savestats", False) | |
171 profileOutput = options.get("profile", None) | |
172 self.profiler = options.get("profiler", None) | |
173 if options.get("nothotshot", False): | |
174 warnings.warn("The --nothotshot option is deprecated. Please " | |
175 "specify the profiler name using the --profiler " | |
176 "option", category=DeprecationWarning) | |
177 self.profiler = "profile" | |
178 if self.profiler in self.profilers: | |
179 profiler = self.profilers[self.profiler](profileOutput, saveStats) | |
180 self.run = profiler.run | |
181 else: | |
182 raise SystemExit("Unsupported profiler name: %s" % (self.profiler,)) | |
183 | |
184 | |
185 | |
186 def runWithProfiler(reactor, config): | |
187 """ | |
188 DEPRECATED in Twisted 8.0. | |
189 | |
190 Run reactor under standard profiler. | |
191 """ | |
192 warnings.warn("runWithProfiler is deprecated since Twisted 8.0. " | |
193 "Use ProfileRunner instead.", DeprecationWarning, 2) | |
194 item = AppProfiler(config) | |
195 return item.run(reactor) | |
196 | |
197 | |
198 | |
199 def runWithHotshot(reactor, config): | |
200 """ | |
201 DEPRECATED in Twisted 8.0. | |
202 | |
203 Run reactor under hotshot profiler. | |
204 """ | |
205 warnings.warn("runWithHotshot is deprecated since Twisted 8.0. " | |
206 "Use HotshotRunner instead.", DeprecationWarning, 2) | |
207 item = AppProfiler(config) | |
208 return item.run(reactor) | |
209 | |
210 | |
211 | |
212 def fixPdb(): | |
213 def do_stop(self, arg): | |
214 self.clear_all_breaks() | |
215 self.set_continue() | |
216 from twisted.internet import reactor | |
217 reactor.callLater(0, reactor.stop) | |
218 return 1 | |
219 | |
220 def help_stop(self): | |
221 print """stop - Continue execution, then cleanly shutdown the twisted re
actor.""" | |
222 | |
223 def set_quit(self): | |
224 os._exit(0) | |
225 | |
226 pdb.Pdb.set_quit = set_quit | |
227 pdb.Pdb.do_stop = do_stop | |
228 pdb.Pdb.help_stop = help_stop | |
229 | |
230 | |
231 | |
232 def runReactorWithLogging(config, oldstdout, oldstderr, profiler=None, reactor=N
one): | |
233 """ | |
234 Start the reactor, using profiling if specified by the configuration, and | |
235 log any error happening in the process. | |
236 | |
237 @param config: configuration of the twistd application. | |
238 @type config: L{ServerOptions} | |
239 | |
240 @param oldstdout: initial value of C{sys.stdout}. | |
241 @type oldstdout: C{file} | |
242 | |
243 @param oldstderr: initial value of C{sys.stderr}. | |
244 @type oldstderr: C{file} | |
245 | |
246 @param profiler: object used to run the reactor with profiling. | |
247 @type profiler: L{AppProfiler} | |
248 | |
249 @param reactor: The reactor to use. If C{None}, the global reactor will | |
250 be used. | |
251 """ | |
252 if reactor is None: | |
253 from twisted.internet import reactor | |
254 try: | |
255 if config['profile']: | |
256 if profiler is not None: | |
257 profiler.run(reactor) | |
258 else: | |
259 # Backward compatible code | |
260 if not config['nothotshot']: | |
261 runWithHotshot(reactor, config) | |
262 else: | |
263 runWithProfiler(reactor, config) | |
264 elif config['debug']: | |
265 sys.stdout = oldstdout | |
266 sys.stderr = oldstderr | |
267 if runtime.platformType == 'posix': | |
268 signal.signal(signal.SIGUSR2, lambda *args: pdb.set_trace()) | |
269 signal.signal(signal.SIGINT, lambda *args: pdb.set_trace()) | |
270 fixPdb() | |
271 pdb.runcall(reactor.run) | |
272 else: | |
273 reactor.run() | |
274 except: | |
275 if config['nodaemon']: | |
276 file = oldstdout | |
277 else: | |
278 file = open("TWISTD-CRASH.log",'a') | |
279 traceback.print_exc(file=file) | |
280 file.flush() | |
281 | |
282 | |
283 | |
284 def getPassphrase(needed): | |
285 if needed: | |
286 return getpass.getpass('Passphrase: ') | |
287 else: | |
288 return None | |
289 | |
290 | |
291 | |
292 def getSavePassphrase(needed): | |
293 if needed: | |
294 passphrase = util.getPassword("Encryption passphrase: ") | |
295 else: | |
296 return None | |
297 | |
298 | |
299 | |
300 class ApplicationRunner(object): | |
301 """ | |
302 An object which helps running an application based on a config object. | |
303 | |
304 Subclass me and implement preApplication and postApplication | |
305 methods. postApplication generally will want to run the reactor | |
306 after starting the application. | |
307 | |
308 @ivar config: The config object, which provides a dict-like interface. | |
309 | |
310 @ivar application: Available in postApplication, but not | |
311 preApplication. This is the application object. | |
312 | |
313 @ivar profilerFactory: Factory for creating a profiler object, able to | |
314 profile the application if options are set accordingly. | |
315 | |
316 @ivar profiler: Instance provided by C{profilerFactory}. | |
317 """ | |
318 profilerFactory = AppProfiler | |
319 | |
320 def __init__(self, config): | |
321 self.config = config | |
322 self.profiler = self.profilerFactory(config) | |
323 | |
324 | |
325 def run(self): | |
326 """ | |
327 Run the application. | |
328 """ | |
329 self.preApplication() | |
330 self.application = self.createOrGetApplication() | |
331 | |
332 # Later, try adapting self.application to ILogObserverFactory or | |
333 # whatever and getting an observer from it, instead. Fall back to | |
334 # self.getLogObserver if the adaption fails. | |
335 self.startLogging(self.getLogObserver()) | |
336 | |
337 self.postApplication() | |
338 | |
339 | |
340 def startReactor(self, reactor, oldstdout, oldstderr): | |
341 """ | |
342 Run the reactor with the given configuration. Subclasses should | |
343 probably call this from C{postApplication}. | |
344 | |
345 @see: L{runReactorWithLogging} | |
346 """ | |
347 runReactorWithLogging( | |
348 self.config, oldstdout, oldstderr, self.profiler, reactor) | |
349 | |
350 | |
351 def preApplication(self): | |
352 """ | |
353 Override in subclass. | |
354 | |
355 This should set up any state necessary before loading and | |
356 running the Application. | |
357 """ | |
358 raise NotImplementedError() | |
359 | |
360 | |
361 def startLogging(self, observer): | |
362 """ | |
363 Initialize the logging system. | |
364 | |
365 @param observer: The observer to add to the logging system. | |
366 """ | |
367 log.startLoggingWithObserver(observer) | |
368 sys.stdout.flush() | |
369 initialLog() | |
370 | |
371 | |
372 def getLogObserver(self): | |
373 """ | |
374 Create a log observer to be added to the logging system before running | |
375 this application. | |
376 """ | |
377 raise NotImplementedError() | |
378 | |
379 | |
380 def postApplication(self): | |
381 """ | |
382 Override in subclass. | |
383 | |
384 This will be called after the application has been loaded (so | |
385 the C{application} attribute will be set). Generally this | |
386 should start the application and run the reactor. | |
387 """ | |
388 raise NotImplementedError | |
389 | |
390 | |
391 def createOrGetApplication(self): | |
392 """ | |
393 Create or load an Application based on the parameters found in the | |
394 given L{ServerOptions} instance. | |
395 | |
396 If a subcommand was used, the L{service.IServiceMaker} that it | |
397 represents will be used to construct a service to be added to | |
398 a newly-created Application. | |
399 | |
400 Otherwise, an application will be loaded based on parameters in | |
401 the config. | |
402 """ | |
403 if self.config.subCommand: | |
404 # If a subcommand was given, it's our responsibility to create | |
405 # the application, instead of load it from a file. | |
406 | |
407 # loadedPlugins is set up by the ServerOptions.subCommands | |
408 # property, which is iterated somewhere in the bowels of | |
409 # usage.Options. | |
410 plg = self.config.loadedPlugins[self.config.subCommand] | |
411 ser = plg.makeService(self.config.subOptions) | |
412 application = service.Application(plg.tapname) | |
413 ser.setServiceParent(application) | |
414 else: | |
415 passphrase = getPassphrase(self.config['encrypted']) | |
416 application = getApplication(self.config, passphrase) | |
417 return application | |
418 | |
419 | |
420 | |
421 def getApplication(config, passphrase): | |
422 s = [(config[t], t) | |
423 for t in ['python', 'xml', 'source', 'file'] if config[t]][0] | |
424 filename, style = s[0], {'file':'pickle'}.get(s[1],s[1]) | |
425 try: | |
426 log.msg("Loading %s..." % filename) | |
427 application = service.loadApplication(filename, style, passphrase) | |
428 log.msg("Loaded.") | |
429 except Exception, e: | |
430 s = "Failed to load application: %s" % e | |
431 if isinstance(e, KeyError) and e.args[0] == "application": | |
432 s += """ | |
433 Could not find 'application' in the file. To use 'twistd -y', your .tac | |
434 file must create a suitable object (e.g., by calling service.Application()) | |
435 and store it in a variable named 'application'. twistd loads your .tac file | |
436 and scans the global variables for one of this name. | |
437 | |
438 Please read the 'Using Application' HOWTO for details. | |
439 """ | |
440 traceback.print_exc(file=log.logfile) | |
441 log.msg(s) | |
442 log.deferr() | |
443 sys.exit('\n' + s + '\n') | |
444 return application | |
445 | |
446 | |
447 | |
448 def reportProfile(report_profile, name): | |
449 """ | |
450 DEPRECATED since Twisted 8.0. This does nothing. | |
451 """ | |
452 warnings.warn("reportProfile is deprecated and a no-op since Twisted 8.0.", | |
453 category=DeprecationWarning) | |
454 | |
455 | |
456 | |
457 def _reactorZshAction(): | |
458 return "(%s)" % " ".join([r.shortName for r in reactors.getReactorTypes()]) | |
459 | |
460 class ReactorSelectionMixin: | |
461 """ | |
462 Provides options for selecting a reactor to install. | |
463 """ | |
464 zsh_actions = {"reactor" : _reactorZshAction} | |
465 messageOutput = sys.stdout | |
466 | |
467 | |
468 def opt_help_reactors(self): | |
469 """ | |
470 Display a list of possibly available reactor names. | |
471 """ | |
472 for r in reactors.getReactorTypes(): | |
473 self.messageOutput.write(' %-4s\t%s\n' % | |
474 (r.shortName, r.description)) | |
475 raise SystemExit(0) | |
476 | |
477 | |
478 def opt_reactor(self, shortName): | |
479 """ | |
480 Which reactor to use (see --help-reactors for a list of possibilities) | |
481 """ | |
482 # Actually actually actually install the reactor right at this very | |
483 # moment, before any other code (for example, a sub-command plugin) | |
484 # runs and accidentally imports and installs the default reactor. | |
485 # | |
486 # This could probably be improved somehow. | |
487 try: | |
488 installReactor(shortName) | |
489 except NoSuchReactor: | |
490 msg = ("The specified reactor does not exist: '%s'.\n" | |
491 "See the list of available reactors with " | |
492 "--help-reactors" % (shortName,)) | |
493 raise usage.UsageError(msg) | |
494 except Exception, e: | |
495 msg = ("The specified reactor cannot be used, failed with error: " | |
496 "%s.\nSee the list of available reactors with " | |
497 "--help-reactors" % (e,)) | |
498 raise usage.UsageError(msg) | |
499 opt_r = opt_reactor | |
500 | |
501 | |
502 | |
503 | |
504 class ServerOptions(usage.Options, ReactorSelectionMixin): | |
505 | |
506 optFlags = [['savestats', None, | |
507 "save the Stats object rather than the text output of " | |
508 "the profiler."], | |
509 ['no_save','o', "do not save state on shutdown"], | |
510 ['encrypted', 'e', | |
511 "The specified tap/aos/xml file is encrypted."], | |
512 ['nothotshot', None, | |
513 "DEPRECATED. Don't use the 'hotshot' profiler even if " | |
514 "it's available."]] | |
515 | |
516 optParameters = [['logfile','l', None, | |
517 "log to a specified file, - for stdout"], | |
518 ['profile', 'p', None, | |
519 "Run in profile mode, dumping results to specified file"], | |
520 ['profiler', None, "hotshot", | |
521 "Name of the profiler to use, 'hotshot' or 'profile'."], | |
522 ['file','f','twistd.tap', | |
523 "read the given .tap file"], | |
524 ['python','y', None, | |
525 "read an application from within a Python file (implies -o
)"], | |
526 ['xml', 'x', None, | |
527 "Read an application from a .tax file " | |
528 "(Marmalade format)."], | |
529 ['source', 's', None, | |
530 "Read an application from a .tas file (AOT format)."], | |
531 ['rundir','d','.', | |
532 'Change to a supplied directory before running'], | |
533 ['report-profile', None, None, | |
534 'E-mail address to use when reporting dynamic execution ' | |
535 'profiler stats. This should not be combined with ' | |
536 'other profiling options. This will only take effect ' | |
537 'if the application to be run has an application ' | |
538 'name.']] | |
539 | |
540 #zsh_altArgDescr = {"foo":"use this description for foo instead"} | |
541 #zsh_multiUse = ["foo", "bar"] | |
542 zsh_mutuallyExclusive = [("file", "python", "xml", "source")] | |
543 zsh_actions = {"file":'_files -g "*.tap"', | |
544 "python":'_files -g "*.(tac|py)"', | |
545 "xml":'_files -g "*.tax"', | |
546 "source":'_files -g "*.tas"', | |
547 "rundir":"_dirs"} | |
548 #zsh_actionDescr = {"logfile":"log file name", "random":"random seed"} | |
549 | |
550 def __init__(self, *a, **kw): | |
551 self['debug'] = False | |
552 usage.Options.__init__(self, *a, **kw) | |
553 | |
554 def opt_debug(self): | |
555 """ | |
556 run the application in the Python Debugger (implies nodaemon), | |
557 sending SIGUSR2 will drop into debugger | |
558 """ | |
559 defer.setDebugging(True) | |
560 failure.startDebugMode() | |
561 self['debug'] = True | |
562 opt_b = opt_debug | |
563 | |
564 | |
565 def opt_spew(self): | |
566 """Print an insanely verbose log of everything that happens. | |
567 Useful when debugging freezes or locks in complex code.""" | |
568 sys.settrace(util.spewer) | |
569 try: | |
570 import threading | |
571 except ImportError: | |
572 return | |
573 threading.settrace(util.spewer) | |
574 | |
575 | |
576 def opt_report_profile(self, value): | |
577 """ | |
578 DEPRECATED. | |
579 | |
580 Manage --report-profile option, which does nothing currently. | |
581 """ | |
582 warnings.warn("--report-profile option is deprecated and a no-op " | |
583 "since Twisted 8.0.", category=DeprecationWarning) | |
584 | |
585 | |
586 def parseOptions(self, options=None): | |
587 if options is None: | |
588 options = sys.argv[1:] or ["--help"] | |
589 usage.Options.parseOptions(self, options) | |
590 | |
591 def postOptions(self): | |
592 if self.subCommand or self['python']: | |
593 self['no_save'] = True | |
594 | |
595 def subCommands(self): | |
596 from twisted import plugin | |
597 plugins = plugin.getPlugins(service.IServiceMaker) | |
598 self.loadedPlugins = {} | |
599 for plug in plugins: | |
600 self.loadedPlugins[plug.tapname] = plug | |
601 yield (plug.tapname, None, lambda: plug.options(), plug.description) | |
602 subCommands = property(subCommands) | |
603 | |
604 | |
605 | |
606 def run(runApp, ServerOptions): | |
607 config = ServerOptions() | |
608 try: | |
609 config.parseOptions() | |
610 except usage.error, ue: | |
611 print config | |
612 print "%s: %s" % (sys.argv[0], ue) | |
613 else: | |
614 runApp(config) | |
615 | |
616 | |
617 def initialLog(): | |
618 from twisted.internet import reactor | |
619 log.msg("twistd %s (%s %s) starting up" % (copyright.version, | |
620 sys.executable, | |
621 runtime.shortPythonVersion())) | |
622 log.msg('reactor class: %s' % reactor.__class__) | |
623 | |
624 | |
625 def convertStyle(filein, typein, passphrase, fileout, typeout, encrypt): | |
626 application = service.loadApplication(filein, typein, passphrase) | |
627 sob.IPersistable(application).setStyle(typeout) | |
628 passphrase = getSavePassphrase(encrypt) | |
629 if passphrase: | |
630 fileout = None | |
631 sob.IPersistable(application).save(filename=fileout, passphrase=passphrase) | |
632 | |
633 def startApplication(application, save): | |
634 from twisted.internet import reactor | |
635 service.IService(application).startService() | |
636 if save: | |
637 p = sob.IPersistable(application) | |
638 reactor.addSystemEventTrigger('after', 'shutdown', p.save, 'shutdown') | |
639 reactor.addSystemEventTrigger('before', 'shutdown', | |
640 service.IService(application).stopService) | |
641 | |
642 def getLogFile(logfilename): | |
643 """ | |
644 Build a log file from the full path. | |
645 """ | |
646 import warnings | |
647 warnings.warn( | |
648 "app.getLogFile is deprecated. Use " | |
649 "twisted.python.logfile.LogFile.fromFullPath instead", | |
650 DeprecationWarning, stacklevel=2) | |
651 | |
652 return logfile.LogFile.fromFullPath(logfilename) | |
653 | |
OLD | NEW |