aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['read_text',\r
  21            'write_text',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def read_text(text_file):\r
  28     """\r
  29     Object description.\r
  30 \r
  31     Parameters\r
  32     ----------\r
  33     parameter : type\r
  34         Parameter description.\r
  35 \r
  36     Returns\r
  37     -------\r
  38     type\r
  39          Return value description.\r
  40     """\r
  41 \r
  42     if text_file != '':\r
  43         with open(text_file, 'rb') as fp:\r
  44             text = (fp.read())\r
  45     return text\r
  46 \r
  47 \r
  48 def write_text(text, text_file):\r
  49     """\r
  50     Object description.\r
  51 \r
  52     Parameters\r
  53     ----------\r
  54     parameter : type\r
  55         Parameter description.\r
  56 \r
  57     Returns\r
  58     -------\r
  59     type\r
  60          Return value description.\r
  61     """\r
  62 \r
  63     if text_file != '':\r
  64         with open(text_file, 'wb') as fp:\r
  65             fp.write(text)\r
  66     return text\r
  67 \r
  68 \r
  69 class Process:\r
  70     """\r
  71     A process with logged output.\r
  72     """\r
  73 \r
  74     def __init__(self,\r
  75                  description=None,\r
  76                  cmd=None,\r
  77                  args=[],\r
  78                  cwd=None,\r
  79                  env=None,\r
  80                  batch_wrapper=False):\r
  81         """\r
  82         Initialize the standard class variables.\r
  83 \r
  84         Parameters\r
  85         ----------\r
  86         parameter : type\r
  87             Parameter description.\r
  88 \r
  89         Returns\r
  90         -------\r
  91         type\r
  92              Return value description.\r
  93         """\r
  94 \r
  95         self.cmd = cmd\r
  96         if not description:\r
  97             self.description = cmd\r
  98         else:\r
  99             self.description = description\r
 100         self.status = None\r
 101         self.args = args\r
 102         self.start = None\r
 103         self.end = None\r
 104         self.log = []\r
 105         self.echo = True\r
 106         self.cwd = cwd\r
 107         self.env = env\r
 108         self.batch_wrapper = batch_wrapper\r
 109         self.process_keys = []\r
 110 \r
 111     def get_elapsed_seconds(self):\r
 112         """\r
 113         Object description.\r
 114 \r
 115         Parameters\r
 116         ----------\r
 117         parameter : type\r
 118             Parameter description.\r
 119 \r
 120         Returns\r
 121         -------\r
 122         type\r
 123              Return value description.\r
 124         """\r
 125 \r
 126         import math\r
 127 \r
 128         if self.end and self.start:\r
 129             delta = (self.end - self.start)\r
 130             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 131                                    int(math.floor(delta.microseconds / 1e3)))\r
 132         else:\r
 133             formatted = None\r
 134         return formatted\r
 135 \r
 136     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 137         """\r
 138         Writes a key / value pair in a supported format.\r
 139 \r
 140         Parameters\r
 141         ----------\r
 142         parameter : type\r
 143             Parameter description.\r
 144 \r
 145         Returns\r
 146         -------\r
 147         type\r
 148              Return value description.\r
 149         """\r
 150 \r
 151         if key != None and (value != None or start_stop != None):\r
 152             indent = '\t' * write_dict['indentationLevel']\r
 153             if write_dict['format'] == 'xml':\r
 154                 if start_stop == 'start':\r
 155                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 156                 elif start_stop == 'stop':\r
 157                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 158                 else:\r
 159                     write_dict['logHandle'].write(\r
 160                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 161             else:  # writeDict['format'] == 'txt':\r
 162                 write_dict['logHandle'].write(\r
 163                     '%s%40s : %s\n' % (indent, key, value))\r
 164 \r
 165     def write_log_header(self, write_dict):\r
 166         """\r
 167         Object description.\r
 168 \r
 169         Parameters\r
 170         ----------\r
 171         parameter : type\r
 172             Parameter description.\r
 173 \r
 174         Returns\r
 175         -------\r
 176         type\r
 177              Return value description.\r
 178         """\r
 179 \r
 180         import platform\r
 181 \r
 182         # Retrieve operating environment information\r
 183         user = None\r
 184         try:\r
 185             user = os.getlogin()\r
 186         except:\r
 187             try:\r
 188                 user = os.getenv('USERNAME')\r
 189                 if user is None:\r
 190                     user = os.getenv('USER')\r
 191             except:\r
 192                 user = 'unknown_user'\r
 193         try:\r
 194             (sysname, nodename, release, version, machine,\r
 195              processor) = platform.uname()\r
 196         except:\r
 197             (sysname, nodename, release, version, machine, processor) = (\r
 198                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 199                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 200         try:\r
 201             hostname = platform.node()\r
 202         except:\r
 203             hostname = 'unknown_hostname'\r
 204 \r
 205         self.write_key(write_dict, 'process', None, 'start')\r
 206         write_dict['indentationLevel'] += 1\r
 207 \r
 208         self.write_key(write_dict, 'description', self.description)\r
 209         self.write_key(write_dict, 'cmd', self.cmd)\r
 210         if self.args:\r
 211             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 212         self.write_key(write_dict, 'start', self.start)\r
 213         self.write_key(write_dict, 'end', self.end)\r
 214         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 215 \r
 216         self.write_key(write_dict, 'user', user)\r
 217         self.write_key(write_dict, 'sysname', sysname)\r
 218         self.write_key(write_dict, 'nodename', nodename)\r
 219         self.write_key(write_dict, 'release', release)\r
 220         self.write_key(write_dict, 'version', version)\r
 221         self.write_key(write_dict, 'machine', machine)\r
 222         self.write_key(write_dict, 'processor', processor)\r
 223 \r
 224         if len(self.process_keys) > 0:\r
 225             self.write_key(write_dict, 'processKeys', None, 'start')\r
 226             for pair in self.process_keys:\r
 227                 (key, value) = pair\r
 228                 write_dict['indentationLevel'] += 1\r
 229                 self.write_key(write_dict, key, value)\r
 230                 write_dict['indentationLevel'] -= 1\r
 231             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 232 \r
 233         self.write_key(write_dict, 'status', self.status)\r
 234 \r
 235     def write_log_footer(self, write_dict):\r
 236         """\r
 237         Object description.\r
 238 \r
 239         Parameters\r
 240         ----------\r
 241         parameter : type\r
 242             Parameter description.\r
 243 \r
 244         Returns\r
 245         -------\r
 246         type\r
 247              Return value description.\r
 248         """\r
 249 \r
 250         write_dict['indentationLevel'] -= 1\r
 251         self.write_key(write_dict, 'process', None, 'stop')\r
 252 \r
 253     def write_log(self,\r
 254                   log_handle=sys.stdout,\r
 255                   indentation_level=0,\r
 256                   format='xml'):\r
 257         """\r
 258         Writes logging information to the specified handle.\r
 259 \r
 260         Parameters\r
 261         ----------\r
 262         parameter : type\r
 263             Parameter description.\r
 264 \r
 265         Returns\r
 266         -------\r
 267         type\r
 268              Return value description.\r
 269         """\r
 270 \r
 271         write_dict = {}\r
 272         write_dict['logHandle'] = log_handle\r
 273         write_dict['indentationLevel'] = indentation_level\r
 274         write_dict['format'] = format\r
 275 \r
 276         if log_handle:\r
 277             self.write_log_header(write_dict)\r
 278 \r
 279             if self.log:\r
 280                 self.write_key(write_dict, 'output', None, 'start')\r
 281                 if format == 'xml':\r
 282                     log_handle.write('<![CDATA[\n')\r
 283                 for line in self.log:\r
 284                     log_handle.write('%s%s\n' % ('', line))\r
 285                 if format == 'xml':\r
 286                     log_handle.write(']]>\n')\r
 287                 self.write_key(write_dict, 'output', None, 'stop')\r
 288 \r
 289             self.write_log_footer(write_dict)\r
 290 \r
 291     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 292         """\r
 293         Object description.\r
 294 \r
 295         Parameters\r
 296         ----------\r
 297         parameter : type\r
 298             Parameter description.\r
 299 \r
 300         Returns\r
 301         -------\r
 302         type\r
 303              Return value description.\r
 304         """\r
 305 \r
 306         if log_filename:\r
 307             try:\r
 308                 # TODO: Review statements.\r
 309                 # 3.1\r
 310                 try:\r
 311                     log_handle = (\r
 312                         open(log_filename, mode='wt', encoding='utf-8'))\r
 313                 # 2.6\r
 314                 except:\r
 315                     log_handle = open(log_filename, mode='wt')\r
 316             except:\r
 317                 print('Couldn\'t open log : %s' % log_filename)\r
 318                 log_handle = None\r
 319 \r
 320         if log_handle:\r
 321             if header:\r
 322                 if format == 'xml':\r
 323                     log_handle.write('<![CDATA[\n')\r
 324                 log_handle.write(header)\r
 325                 if format == 'xml':\r
 326                     log_handle.write(']]>\n')\r
 327             self.write_log(log_handle)\r
 328             log_handle.close()\r
 329 \r
 330     def log_line(self, line):\r
 331         """\r
 332         Adds a line of text to the log.\r
 333 \r
 334         Parameters\r
 335         ----------\r
 336         parameter : type\r
 337             Parameter description.\r
 338 \r
 339         Returns\r
 340         -------\r
 341         type\r
 342              Return value description.\r
 343         """\r
 344 \r
 345         self.log.append(line.rstrip())\r
 346         if self.echo:\r
 347             print('%s' % line.rstrip())\r
 348 \r
 349     def execute(self):\r
 350         """\r
 351         Executes the current process.\r
 352 \r
 353         Parameters\r
 354         ----------\r
 355         parameter : type\r
 356             Parameter description.\r
 357 \r
 358         Returns\r
 359         -------\r
 360         type\r
 361              Return value description.\r
 362         """\r
 363 \r
 364         import datetime\r
 365         import traceback\r
 366 \r
 367         try:\r
 368             import subprocess as sp\r
 369         except:\r
 370             sp = None\r
 371 \r
 372         self.start = datetime.datetime.now()\r
 373 \r
 374         cmdargs = [self.cmd]\r
 375         cmdargs.extend(self.args)\r
 376 \r
 377         if self.echo:\r
 378             if sp:\r
 379                 print(\r
 380                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 381             else:\r
 382                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 383 \r
 384         # intialize a few variables that may or may not be set later\r
 385         process = None\r
 386         tmp_wrapper = None\r
 387         stdout = None\r
 388         stdin = None\r
 389         parentenv = os.environ\r
 390         parentcwd = os.getcwd()\r
 391 \r
 392         try:\r
 393             # Using subprocess\r
 394             if sp:\r
 395                 if self.batch_wrapper:\r
 396                     cmd = ' '.join(cmdargs)\r
 397                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 398                     write_text(cmd, tmp_wrapper)\r
 399                     print('%s : Running process through wrapper %s\n' % (\r
 400                         self.__class__, tmp_wrapper))\r
 401                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 402                                        stderr=sp.STDOUT,\r
 403                                        cwd=self.cwd, env=self.env)\r
 404                 else:\r
 405                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 406                                        stderr=sp.STDOUT,\r
 407                                        cwd=self.cwd, env=self.env)\r
 408 \r
 409             # using os.popen4\r
 410             else:\r
 411                 if self.env:\r
 412                     os.environ = self.env\r
 413                 if self.cwd:\r
 414                     os.chdir(self.cwd)\r
 415 \r
 416                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 417         except:\r
 418             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 419             traceback.print_exc()\r
 420 \r
 421         # Using subprocess\r
 422         if sp:\r
 423             if process != None:\r
 424                 # pid = process.pid\r
 425                 # log.logLine('process id %s\n' % pid)\r
 426 \r
 427                 try:\r
 428                     # This is more proper python, and resolves some issues with\r
 429                     # a process ending before all of its output has been\r
 430                     # processed, but it also seems to stall when the read\r
 431                     # buffer is near or over it's limit. this happens\r
 432                     # relatively frequently with processes that generate lots\r
 433                     # of print statements.\r
 434                     for line in process.stdout:\r
 435                         self.log_line(line)\r
 436                     #\r
 437                     # So we go with the, um, uglier  option below\r
 438 \r
 439                     # This is now used to ensure that the process has finished\r
 440                     line = ''\r
 441                     while line != None and process.poll() is None:\r
 442                         try:\r
 443                             line = process.stdout.readline()\r
 444                         except:\r
 445                             break\r
 446                         # 3.1\r
 447                         try:\r
 448                             self.log_line(str(line, encoding='utf-8'))\r
 449                         # 2.6\r
 450                         except:\r
 451                             self.log_line(line)\r
 452                 except:\r
 453                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 454 \r
 455                 self.status = process.returncode\r
 456 \r
 457                 if self.batch_wrapper and tmp_wrapper:\r
 458                     try:\r
 459                         os.remove(tmp_wrapper)\r
 460                     except:\r
 461                         print(\r
 462                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 463                         traceback.print_exc()\r
 464 \r
 465         # Using os.popen4\r
 466         else:\r
 467             exit_code = -1\r
 468             try:\r
 469                 # print('reading stdout lines')\r
 470                 stdout_lines = stdout.readlines()\r
 471                 exit_code = stdout.close()\r
 472 \r
 473                 stdout.close()\r
 474                 stdin.close()\r
 475 \r
 476                 if self.env:\r
 477                     os.environ = parentenv\r
 478                 if self.cwd:\r
 479                     os.chdir(parentcwd)\r
 480 \r
 481                 if len(stdout_lines) > 0:\r
 482                     for line in stdout_lines:\r
 483                         self.log_line(line)\r
 484 \r
 485                 if not exit_code:\r
 486                     exit_code = 0\r
 487             except:\r
 488                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 489 \r
 490             self.status = exit_code\r
 491 \r
 492         self.end = datetime.datetime.now()\r
 493 \r
 494 \r
 495 class ProcessList(Process):\r
 496     """\r
 497     A list of processes with logged output.\r
 498     """\r
 499 \r
 500     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 501         """\r
 502         Object description.\r
 503 \r
 504         Parameters\r
 505         ----------\r
 506         parameter : type\r
 507             Parameter description.\r
 508 \r
 509         Returns\r
 510         -------\r
 511         type\r
 512              Return value description.\r
 513         """\r
 514 \r
 515         Process.__init__(self, description, None, None, cwd, env)\r
 516         'Initialize the standard class variables'\r
 517         self.processes = []\r
 518         self.blocking = blocking\r
 519 \r
 520     def generate_report(self, write_dict):\r
 521         """\r
 522         Generates a log based on the success of the child processes.\r
 523 \r
 524         Parameters\r
 525         ----------\r
 526         parameter : type\r
 527             Parameter description.\r
 528 \r
 529         Returns\r
 530         -------\r
 531         type\r
 532              Return value description.\r
 533         """\r
 534 \r
 535         if self.processes:\r
 536             _status = True\r
 537             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 538 \r
 539             self.log = []\r
 540 \r
 541             for child in self.processes:\r
 542                 if isinstance(child, ProcessList):\r
 543                     child.generate_report(write_dict)\r
 544 \r
 545                 child_result = ''\r
 546                 key = child.description\r
 547                 value = child.status\r
 548                 if write_dict['format'] == 'xml':\r
 549                     child_result = (\r
 550                         '%s<result description=\'%s\'>%s</result>' % (\r
 551                             indent, key, value))\r
 552                 else:  # writeDict['format'] == 'txt':\r
 553                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 554                 self.log.append(child_result)\r
 555 \r
 556                 if child.status != 0:\r
 557                     _status = False\r
 558             if not _status:\r
 559                 self.status = -1\r
 560             else:\r
 561                 self.status = 0\r
 562         else:\r
 563             self.log = ['No child processes available to generate a report']\r
 564             self.status = -1\r
 565 \r
 566     def write_log_header(self, write_dict):\r
 567         """\r
 568         Object description.\r
 569 \r
 570         Parameters\r
 571         ----------\r
 572         parameter : type\r
 573             Parameter description.\r
 574 \r
 575         Returns\r
 576         -------\r
 577         type\r
 578              Return value description.\r
 579         """\r
 580 \r
 581         self.write_key(write_dict, 'processList', None, 'start')\r
 582         write_dict['indentationLevel'] += 1\r
 583 \r
 584         self.write_key(write_dict, 'description', self.description)\r
 585         self.write_key(write_dict, 'start', self.start)\r
 586         self.write_key(write_dict, 'end', self.end)\r
 587         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 588 \r
 589         self.generate_report(write_dict)\r
 590 \r
 591         self.write_key(write_dict, 'status', self.status)\r
 592 \r
 593     def write_log_footer(self, write_dict):\r
 594         """\r
 595         Object description.\r
 596 \r
 597         Parameters\r
 598         ----------\r
 599         parameter : type\r
 600             Parameter description.\r
 601 \r
 602         Returns\r
 603         -------\r
 604         type\r
 605              Return value description.\r
 606         """\r
 607 \r
 608         write_dict['indentationLevel'] -= 1\r
 609         self.write_key(write_dict, 'processList', None, 'stop')\r
 610 \r
 611     def write_log(self,\r
 612                   log_handle=sys.stdout,\r
 613                   indentation_level=0,\r
 614                   format='xml'):\r
 615         """\r
 616         Writes logging information to the specified handle.\r
 617 \r
 618         Parameters\r
 619         ----------\r
 620         parameter : type\r
 621             Parameter description.\r
 622 \r
 623         Returns\r
 624         -------\r
 625         type\r
 626              Return value description.\r
 627         """\r
 628 \r
 629         write_dict = {}\r
 630         write_dict['logHandle'] = log_handle\r
 631         write_dict['indentationLevel'] = indentation_level\r
 632         write_dict['format'] = format\r
 633 \r
 634         if log_handle:\r
 635             self.write_log_header(write_dict)\r
 636 \r
 637             if self.log:\r
 638                 self.write_key(write_dict, 'output', None, 'start')\r
 639                 for line in self.log:\r
 640                     log_handle.write('%s%s\n' % ('', line))\r
 641                 self.write_key(write_dict, 'output', None, 'stop')\r
 642 \r
 643             if self.processes:\r
 644                 self.write_key(write_dict, 'processes', None, 'start')\r
 645                 for child in self.processes:\r
 646                     child.write_log(log_handle, indentation_level + 1, format)\r
 647                 self.write_key(write_dict, 'processes', None, 'stop')\r
 648 \r
 649             self.write_log_footer(write_dict)\r
 650 \r
 651     def execute(self):\r
 652         """\r
 653         Executes the list of processes.\r
 654 \r
 655         Parameters\r
 656         ----------\r
 657         parameter : type\r
 658             Parameter description.\r
 659 \r
 660         Returns\r
 661         -------\r
 662         type\r
 663              Return value description.\r
 664         """\r
 665 \r
 666         import datetime\r
 667 \r
 668         self.start = datetime.datetime.now()\r
 669 \r
 670         self.status = 0\r
 671         if self.processes:\r
 672             for child in self.processes:\r
 673                 if child:\r
 674                     try:\r
 675                         child.execute()\r
 676                     except:\r
 677                         print('%s : caught exception in child class %s' % (\r
 678                             self.__class__, child.__class__))\r
 679                         traceback.print_exc()\r
 680                         child.status = -1\r
 681 \r
 682                     if self.blocking and child.status != 0:\r
 683                         print('%s : child class %s finished with an error' % (\r
 684                             self.__class__, child.__class__))\r
 685                         self.status = -1\r
 686                         break\r
 687 \r
 688         self.end = datetime.datetime.now()\r
 689 \r
 690 \r
 691 def main():\r
 692     """\r
 693     Object description.\r
 694 \r
 695     Parameters\r
 696     ----------\r
 697     parameter : type\r
 698         Parameter description.\r
 699 \r
 700     Returns\r
 701     -------\r
 702     type\r
 703          Return value description.\r
 704     """\r
 705 \r
 706     import optparse\r
 707 \r
 708     p = optparse.OptionParser(description='A process logging script',\r
 709                               prog='process',\r
 710                               version='process 0.1',\r
 711                               usage=('%prog [options] '\r
 712                                      '[options for the logged process]'))\r
 713     p.add_option('--cmd', '-c', default=None)\r
 714     p.add_option('--log', '-l', default=None)\r
 715 \r
 716     options, arguments = p.parse_args()\r
 717 \r
 718     #\r
 719     # Get options\r
 720     # \r
 721     cmd = options.cmd\r
 722     log_filename = options.log\r
 723 \r
 724     try:\r
 725         args_start = sys.argv.index('--') + 1\r
 726         args = sys.argv[args_start:]\r
 727     except:\r
 728         args_start = len(sys.argv) + 1\r
 729         args = []\r
 730 \r
 731     if cmd is None:\r
 732         print('process: No command specified')\r
 733 \r
 734     #\r
 735     # Test regular logging\r
 736     #\r
 737     process = Process(description='a process', cmd=cmd, args=args)\r
 738 \r
 739     #\r
 740     # Test report generation and writing a log\r
 741     #\r
 742     process_list = ProcessList('a process list')\r
 743     process_list.processes.append(process)\r
 744     process_list.echo = True\r
 745     process_list.execute()\r
 746 \r
 747     process_list.write_log_to_disk(log_filename)\r
 748 \r
 749 \r
 750 if __name__ == '__main__':\r
 751     main()\r