1 #!/usr/bin/env python
\r
2 # -*- coding: utf-8 -*-
\r
5 A process wrapper class that maintains the text output and execution status of
\r
6 a process or a list of other process wrappers which carry such data.
\r
9 from __future__ import division
\r
15 __author__ = 'ACES Developers'
\r
16 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'
\r
18 __maintainer__ = 'ACES Developers'
\r
19 __email__ = 'aces@oscars.org'
\r
20 __status__ = 'Production'
\r
22 __all__ = ['read_text',
\r
29 def read_text(text_file):
\r
36 Parameter description.
\r
41 Return value description.
\r
45 with open(text_file, 'rb') as fp:
\r
50 def write_text(text, text_file):
\r
57 Parameter description.
\r
62 Return value description.
\r
66 with open(text_file, 'wb') as fp:
\r
73 A process with logged output.
\r
82 batch_wrapper=False):
\r
84 Initialize the standard class variables.
\r
89 Parameter description.
\r
94 Return value description.
\r
101 if not description:
\r
102 self.description = cmd
\r
104 self.description = description
\r
113 self.batch_wrapper = batch_wrapper
\r
114 self.process_keys = []
\r
116 def get_elapsed_seconds(self):
\r
118 Object description.
\r
123 Parameter description.
\r
128 Return value description.
\r
133 if self.end and self.start:
\r
134 delta = (self.end - self.start)
\r
135 formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,
\r
136 int(math.floor(delta.microseconds / 1e3)))
\r
141 def write_key(self, write_dict, key=None, value=None, start_stop=None):
\r
143 Writes a key / value pair in a supported format.
\r
148 Parameter description.
\r
153 Return value description.
\r
156 if key is not None and (value is not None or start_stop is not None):
\r
157 indent = '\t' * write_dict['indentationLevel']
\r
158 if write_dict['format'] == 'xml':
\r
159 if start_stop == 'start':
\r
160 write_dict['logHandle'].write('%s<%s>\n' % (indent, key))
\r
161 elif start_stop == 'stop':
\r
162 write_dict['logHandle'].write('%s</%s>\n' % (indent, key))
\r
164 write_dict['logHandle'].write(
\r
165 '%s<%s>%s</%s>\n' % (indent, key, value, key))
\r
167 write_dict['logHandle'].write(
\r
168 '%s%40s : %s\n' % (indent, key, value))
\r
170 def write_log_header(self, write_dict):
\r
172 Object description.
\r
177 Parameter description.
\r
182 Return value description.
\r
188 user = os.getlogin()
\r
191 user = os.getenv('USERNAME')
\r
193 user = os.getenv('USER')
\r
195 user = 'unknown_user'
\r
197 (sysname, nodename, release, version, machine,
\r
198 processor) = platform.uname()
\r
200 (sysname, nodename, release, version, machine, processor) = (
\r
201 'unknown_sysname', 'unknown_nodename', 'unknown_release',
\r
202 'unknown_version', 'unknown_machine', 'unknown_processor')
\r
204 self.write_key(write_dict, 'process', None, 'start')
\r
205 write_dict['indentationLevel'] += 1
\r
207 self.write_key(write_dict, 'description', self.description)
\r
208 self.write_key(write_dict, 'cmd', self.cmd)
\r
210 self.write_key(write_dict, 'args', ' '.join(self.args))
\r
211 self.write_key(write_dict, 'start', self.start)
\r
212 self.write_key(write_dict, 'end', self.end)
\r
213 self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())
\r
215 self.write_key(write_dict, 'user', user)
\r
216 self.write_key(write_dict, 'sysname', sysname)
\r
217 self.write_key(write_dict, 'nodename', nodename)
\r
218 self.write_key(write_dict, 'release', release)
\r
219 self.write_key(write_dict, 'version', version)
\r
220 self.write_key(write_dict, 'machine', machine)
\r
221 self.write_key(write_dict, 'processor', processor)
\r
223 if len(self.process_keys) > 0:
\r
224 self.write_key(write_dict, 'processKeys', None, 'start')
\r
225 for pair in self.process_keys:
\r
226 (key, value) = pair
\r
227 write_dict['indentationLevel'] += 1
\r
228 self.write_key(write_dict, key, value)
\r
229 write_dict['indentationLevel'] -= 1
\r
230 self.write_key(write_dict, 'processKeys', None, 'stop')
\r
232 self.write_key(write_dict, 'status', self.status)
\r
234 def write_log_footer(self, write_dict):
\r
236 Object description.
\r
241 Parameter description.
\r
246 Return value description.
\r
249 write_dict['indentationLevel'] -= 1
\r
250 self.write_key(write_dict, 'process', None, 'stop')
\r
252 def write_log(self,
\r
253 log_handle=sys.stdout,
\r
254 indentation_level=0,
\r
257 Writes logging information to the specified handle.
\r
262 Parameter description.
\r
267 Return value description.
\r
271 write_dict['logHandle'] = log_handle
\r
272 write_dict['indentationLevel'] = indentation_level
\r
273 write_dict['format'] = format
\r
276 self.write_log_header(write_dict)
\r
279 self.write_key(write_dict, 'output', None, 'start')
\r
280 if format == 'xml':
\r
281 log_handle.write('<![CDATA[\n')
\r
282 for line in self.log:
\r
283 log_handle.write('%s%s\n' % ('', line))
\r
284 if format == 'xml':
\r
285 log_handle.write(']]>\n')
\r
286 self.write_key(write_dict, 'output', None, 'stop')
\r
288 self.write_log_footer(write_dict)
\r
290 def write_log_to_disk(self, log_filename=None, format='xml', header=None):
\r
292 Object description.
\r
297 Parameter description.
\r
302 Return value description.
\r
307 # TODO: Review statements.
\r
311 open(log_filename, mode='wt', encoding='utf-8'))
\r
314 log_handle = open(log_filename, mode='wt')
\r
316 print('Couldn\'t open log : %s' % log_filename)
\r
321 if format == 'xml':
\r
322 log_handle.write('<![CDATA[\n')
\r
323 log_handle.write(header)
\r
324 if format == 'xml':
\r
325 log_handle.write(']]>\n')
\r
326 self.write_log(log_handle)
\r
329 def log_line(self, line):
\r
331 Adds a line of text to the log.
\r
336 Parameter description.
\r
341 Return value description.
\r
344 self.log.append(line.rstrip())
\r
346 print('%s' % line.rstrip())
\r
350 Executes the current process.
\r
355 Parameter description.
\r
360 Return value description.
\r
367 import subprocess as sp
\r
371 self.start = datetime.datetime.now()
\r
373 cmdargs = [self.cmd]
\r
374 cmdargs.extend(self.args)
\r
379 '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))
\r
381 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))
\r
387 parentenv = os.environ
\r
388 parentcwd = os.getcwd()
\r
391 # Using *subprocess*.
\r
393 if self.batch_wrapper:
\r
394 cmd = ' '.join(cmdargs)
\r
395 tmp_wrapper = os.path.join(self.cwd, 'process.bat')
\r
396 write_text(cmd, tmp_wrapper)
\r
397 print('%s : Running process through wrapper %s\n' % (
\r
398 self.__class__, tmp_wrapper))
\r
399 process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,
\r
401 cwd=self.cwd, env=self.env)
\r
403 process = sp.Popen(cmdargs, stdout=sp.PIPE,
\r
405 cwd=self.cwd, env=self.env)
\r
407 # using *os.popen4*.
\r
410 os.environ = self.env
\r
414 stdin, stdout = os.popen4(cmdargs, 'r')
\r
416 print('Couldn\'t execute command : %s' % cmdargs[0])
\r
417 traceback.print_exc()
\r
419 # Using *subprocess*
\r
421 if process is not None:
\r
422 # pid = process.pid
\r
423 # log.logLine('process id %s\n' % pid)
\r
426 # This is more proper python, and resolves some issues with
\r
427 # a process ending before all of its output has been
\r
428 # processed, but it also seems to stall when the read
\r
429 # buffer is near or over its limit. This happens
\r
430 # relatively frequently with processes that generate lots
\r
431 # of print statements.
\r
432 for line in process.stdout:
\r
433 self.log_line(line)
\r
435 # So we go with the, um, uglier option below.
\r
437 # This is now used to ensure that the process has finished.
\r
439 while line is not None and process.poll() is None:
\r
441 line = process.stdout.readline()
\r
446 # TODO: Investigate previous eroneous statement.
\r
447 # self.log_line(str(line, encoding='utf-8'))
\r
448 self.log_line(str(line))
\r
451 self.log_line(line)
\r
453 self.log_line('Logging error : %s' % sys.exc_info()[0])
\r
455 self.status = process.returncode
\r
457 if self.batch_wrapper and tmp_wrapper:
\r
459 os.remove(tmp_wrapper)
\r
462 'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)
\r
463 traceback.print_exc()
\r
465 # Using *os.popen4*.
\r
469 stdout_lines = stdout.readlines()
\r
470 # TODO: Investigate if this is the good behavior, close() does
\r
471 # not return anything / None.
\r
472 exit_code = stdout.close()
\r
478 os.environ = parentenv
\r
480 os.chdir(parentcwd)
\r
482 if len(stdout_lines) > 0:
\r
483 for line in stdout_lines:
\r
484 self.log_line(line)
\r
489 self.log_line('Logging error : %s' % sys.exc_info()[0])
\r
491 self.status = exit_code
\r
493 self.end = datetime.datetime.now()
\r
496 class ProcessList(Process):
\r
498 A list of processes with logged output.
\r
501 def __init__(self, description, blocking=True, cwd=None, env=None):
\r
503 Object description.
\r
508 Parameter description.
\r
513 Return value description.
\r
516 Process.__init__(self, description, None, None, cwd, env)
\r
517 'Initialize the standard class variables'
\r
518 self.processes = []
\r
519 self.blocking = blocking
\r
521 def generate_report(self, write_dict):
\r
523 Generates a log based on the success of the child processes.
\r
528 Parameter description.
\r
533 Return value description.
\r
538 indent = '\t' * (write_dict['indentationLevel'] + 1)
\r
542 for child in self.processes:
\r
543 if isinstance(child, ProcessList):
\r
544 child.generate_report(write_dict)
\r
546 key = child.description
\r
547 value = child.status
\r
548 if write_dict['format'] == 'xml':
\r
550 '%s<result description=\'%s\'>%s</result>' % (
\r
551 indent, key, value))
\r
553 child_result = ('%s%40s : %s' % (indent, key, value))
\r
554 self.log.append(child_result)
\r
556 if child.status != 0:
\r
563 self.log = ['No child processes available to generate a report']
\r
566 def write_log_header(self, write_dict):
\r
568 Object description.
\r
573 Parameter description.
\r
578 Return value description.
\r
581 self.write_key(write_dict, 'processList', None, 'start')
\r
582 write_dict['indentationLevel'] += 1
\r
584 self.write_key(write_dict, 'description', self.description)
\r
585 self.write_key(write_dict, 'start', self.start)
\r
586 self.write_key(write_dict, 'end', self.end)
\r
587 self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())
\r
589 self.generate_report(write_dict)
\r
591 self.write_key(write_dict, 'status', self.status)
\r
593 def write_log_footer(self, write_dict):
\r
595 Object description.
\r
600 Parameter description.
\r
605 Return value description.
\r
608 write_dict['indentationLevel'] -= 1
\r
609 self.write_key(write_dict, 'processList', None, 'stop')
\r
611 def write_log(self,
\r
612 log_handle=sys.stdout,
\r
613 indentation_level=0,
\r
616 Writes logging information to the specified handle.
\r
621 Parameter description.
\r
626 Return value description.
\r
630 write_dict['logHandle'] = log_handle
\r
631 write_dict['indentationLevel'] = indentation_level
\r
632 write_dict['format'] = format
\r
635 self.write_log_header(write_dict)
\r
638 self.write_key(write_dict, 'output', None, 'start')
\r
639 for line in self.log:
\r
640 log_handle.write('%s%s\n' % ('', line))
\r
641 self.write_key(write_dict, 'output', None, 'stop')
\r
644 self.write_key(write_dict, 'processes', None, 'start')
\r
645 for child in self.processes:
\r
646 child.write_log(log_handle, indentation_level + 1, format)
\r
647 self.write_key(write_dict, 'processes', None, 'stop')
\r
649 self.write_log_footer(write_dict)
\r
653 Executes the list of processes.
\r
658 Parameter description.
\r
663 Return value description.
\r
668 self.start = datetime.datetime.now()
\r
672 for child in self.processes:
\r
677 print('%s : caught exception in child class %s' % (
\r
678 self.__class__, child.__class__))
\r
679 traceback.print_exc()
\r
682 if self.blocking and child.status != 0:
\r
683 print('%s : child class %s finished with an error' % (
\r
684 self.__class__, child.__class__))
\r
688 self.end = datetime.datetime.now()
\r
693 Object description.
\r
698 Parameter description.
\r
703 Return value description.
\r
708 p = optparse.OptionParser(description='A process logging script',
\r
710 version='process 0.1',
\r
711 usage=('%prog [options] '
\r
712 '[options for the logged process]'))
\r
713 p.add_option('--cmd', '-c', default=None)
\r
714 p.add_option('--log', '-l', default=None)
\r
716 options, arguments = p.parse_args()
\r
719 log_filename = options.log
\r
722 args_start = sys.argv.index('--') + 1
\r
723 args = sys.argv[args_start:]
\r
725 args_start = len(sys.argv) + 1
\r
729 print('process: No command specified')
\r
731 # Testing regular logging.
\r
732 process = Process(description='a process', cmd=cmd, args=args)
\r
734 # Testing report generation and writing a log.
\r
735 process_list = ProcessList('a process list')
\r
736 process_list.processes.append(process)
\r
737 process_list.echo = True
\r
738 process_list.execute()
\r
740 process_list.write_log_to_disk(log_filename)
\r
743 if __name__ == '__main__':
\r