aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['read_text',\r
  21            'write_text',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def read_text(text_file):\r
  28     """\r
  29     Object description.\r
  30 \r
  31     Parameters\r
  32     ----------\r
  33     parameter : type\r
  34         Parameter description.\r
  35 \r
  36     Returns\r
  37     -------\r
  38     type\r
  39          Return value description.\r
  40     """\r
  41 \r
  42     if text_file != '':\r
  43         with open(text_file, 'rb') as fp:\r
  44             text = (fp.read())\r
  45     return text\r
  46 \r
  47 \r
  48 def write_text(text, text_file):\r
  49     """\r
  50     Object description.\r
  51 \r
  52     Parameters\r
  53     ----------\r
  54     parameter : type\r
  55         Parameter description.\r
  56 \r
  57     Returns\r
  58     -------\r
  59     type\r
  60          Return value description.\r
  61     """\r
  62 \r
  63     if text_file != '':\r
  64         with open(text_file, 'wb') as fp:\r
  65             fp.write(text)\r
  66     return text\r
  67 \r
  68 \r
  69 class Process:\r
  70     """\r
  71     A process with logged output.\r
  72     """\r
  73 \r
  74     def __init__(self,\r
  75                  description=None,\r
  76                  cmd=None,\r
  77                  args=None,\r
  78                  cwd=None,\r
  79                  env=None,\r
  80                  batch_wrapper=False):\r
  81         """\r
  82         Initialize the standard class variables.\r
  83 \r
  84         Parameters\r
  85         ----------\r
  86         parameter : type\r
  87             Parameter description.\r
  88 \r
  89         Returns\r
  90         -------\r
  91         type\r
  92              Return value description.\r
  93         """\r
  94 \r
  95         if args is None:\r
  96             args = []\r
  97 \r
  98         self.cmd = cmd\r
  99         if not description:\r
 100             self.description = cmd\r
 101         else:\r
 102             self.description = description\r
 103         self.status = None\r
 104         self.args = args\r
 105         self.start = None\r
 106         self.end = None\r
 107         self.log = []\r
 108         self.echo = True\r
 109         self.cwd = cwd\r
 110         self.env = env\r
 111         self.batch_wrapper = batch_wrapper\r
 112         self.process_keys = []\r
 113 \r
 114     def get_elapsed_seconds(self):\r
 115         """\r
 116         Object description.\r
 117 \r
 118         Parameters\r
 119         ----------\r
 120         parameter : type\r
 121             Parameter description.\r
 122 \r
 123         Returns\r
 124         -------\r
 125         type\r
 126              Return value description.\r
 127         """\r
 128 \r
 129         import math\r
 130 \r
 131         if self.end and self.start:\r
 132             delta = (self.end - self.start)\r
 133             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 134                                    int(math.floor(delta.microseconds / 1e3)))\r
 135         else:\r
 136             formatted = None\r
 137         return formatted\r
 138 \r
 139     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 140         """\r
 141         Writes a key / value pair in a supported format.\r
 142 \r
 143         Parameters\r
 144         ----------\r
 145         parameter : type\r
 146             Parameter description.\r
 147 \r
 148         Returns\r
 149         -------\r
 150         type\r
 151              Return value description.\r
 152         """\r
 153 \r
 154         if key is not None and (value is not None or start_stop is not None):\r
 155             indent = '\t' * write_dict['indentationLevel']\r
 156             if write_dict['format'] == 'xml':\r
 157                 if start_stop == 'start':\r
 158                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 159                 elif start_stop == 'stop':\r
 160                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 161                 else:\r
 162                     write_dict['logHandle'].write(\r
 163                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 164             else:\r
 165                 write_dict['logHandle'].write(\r
 166                     '%s%40s : %s\n' % (indent, key, value))\r
 167 \r
 168     def write_log_header(self, write_dict):\r
 169         """\r
 170         Object description.\r
 171 \r
 172         Parameters\r
 173         ----------\r
 174         parameter : type\r
 175             Parameter description.\r
 176 \r
 177         Returns\r
 178         -------\r
 179         type\r
 180              Return value description.\r
 181         """\r
 182 \r
 183         import platform\r
 184 \r
 185         try:\r
 186             user = os.getlogin()\r
 187         except:\r
 188             try:\r
 189                 user = os.getenv('USERNAME')\r
 190                 if user is None:\r
 191                     user = os.getenv('USER')\r
 192             except:\r
 193                 user = 'unknown_user'\r
 194         try:\r
 195             (sysname, nodename, release, version, machine,\r
 196              processor) = platform.uname()\r
 197         except:\r
 198             (sysname, nodename, release, version, machine, processor) = (\r
 199                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 200                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 201 \r
 202         self.write_key(write_dict, 'process', None, 'start')\r
 203         write_dict['indentationLevel'] += 1\r
 204 \r
 205         self.write_key(write_dict, 'description', self.description)\r
 206         self.write_key(write_dict, 'cmd', self.cmd)\r
 207         if self.args:\r
 208             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 209         self.write_key(write_dict, 'start', self.start)\r
 210         self.write_key(write_dict, 'end', self.end)\r
 211         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 212 \r
 213         self.write_key(write_dict, 'user', user)\r
 214         self.write_key(write_dict, 'sysname', sysname)\r
 215         self.write_key(write_dict, 'nodename', nodename)\r
 216         self.write_key(write_dict, 'release', release)\r
 217         self.write_key(write_dict, 'version', version)\r
 218         self.write_key(write_dict, 'machine', machine)\r
 219         self.write_key(write_dict, 'processor', processor)\r
 220 \r
 221         if len(self.process_keys) > 0:\r
 222             self.write_key(write_dict, 'processKeys', None, 'start')\r
 223             for pair in self.process_keys:\r
 224                 (key, value) = pair\r
 225                 write_dict['indentationLevel'] += 1\r
 226                 self.write_key(write_dict, key, value)\r
 227                 write_dict['indentationLevel'] -= 1\r
 228             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 229 \r
 230         self.write_key(write_dict, 'status', self.status)\r
 231 \r
 232     def write_log_footer(self, write_dict):\r
 233         """\r
 234         Object description.\r
 235 \r
 236         Parameters\r
 237         ----------\r
 238         parameter : type\r
 239             Parameter description.\r
 240 \r
 241         Returns\r
 242         -------\r
 243         type\r
 244              Return value description.\r
 245         """\r
 246 \r
 247         write_dict['indentationLevel'] -= 1\r
 248         self.write_key(write_dict, 'process', None, 'stop')\r
 249 \r
 250     def write_log(self,\r
 251                   log_handle=sys.stdout,\r
 252                   indentation_level=0,\r
 253                   format='xml'):\r
 254         """\r
 255         Writes logging information to the specified handle.\r
 256 \r
 257         Parameters\r
 258         ----------\r
 259         parameter : type\r
 260             Parameter description.\r
 261 \r
 262         Returns\r
 263         -------\r
 264         type\r
 265              Return value description.\r
 266         """\r
 267 \r
 268         write_dict = {}\r
 269         write_dict['logHandle'] = log_handle\r
 270         write_dict['indentationLevel'] = indentation_level\r
 271         write_dict['format'] = format\r
 272 \r
 273         if log_handle:\r
 274             self.write_log_header(write_dict)\r
 275 \r
 276             if self.log:\r
 277                 self.write_key(write_dict, 'output', None, 'start')\r
 278                 if format == 'xml':\r
 279                     log_handle.write('<![CDATA[\n')\r
 280                 for line in self.log:\r
 281                     log_handle.write('%s%s\n' % ('', line))\r
 282                 if format == 'xml':\r
 283                     log_handle.write(']]>\n')\r
 284                 self.write_key(write_dict, 'output', None, 'stop')\r
 285 \r
 286             self.write_log_footer(write_dict)\r
 287 \r
 288     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 289         """\r
 290         Object description.\r
 291 \r
 292         Parameters\r
 293         ----------\r
 294         parameter : type\r
 295             Parameter description.\r
 296 \r
 297         Returns\r
 298         -------\r
 299         type\r
 300              Return value description.\r
 301         """\r
 302 \r
 303         if log_filename:\r
 304             try:\r
 305                 # TODO: Review statements.\r
 306                 # 3.1\r
 307                 try:\r
 308                     log_handle = (\r
 309                         open(log_filename, mode='wt', encoding='utf-8'))\r
 310                 # 2.6\r
 311                 except:\r
 312                     log_handle = open(log_filename, mode='wt')\r
 313             except:\r
 314                 print('Couldn\'t open log : %s' % log_filename)\r
 315                 log_handle = None\r
 316 \r
 317         if log_handle:\r
 318             if header:\r
 319                 if format == 'xml':\r
 320                     log_handle.write('<![CDATA[\n')\r
 321                 log_handle.write(header)\r
 322                 if format == 'xml':\r
 323                     log_handle.write(']]>\n')\r
 324             self.write_log(log_handle)\r
 325             log_handle.close()\r
 326 \r
 327     def log_line(self, line):\r
 328         """\r
 329         Adds a line of text to the log.\r
 330 \r
 331         Parameters\r
 332         ----------\r
 333         parameter : type\r
 334             Parameter description.\r
 335 \r
 336         Returns\r
 337         -------\r
 338         type\r
 339              Return value description.\r
 340         """\r
 341 \r
 342         self.log.append(line.rstrip())\r
 343         if self.echo:\r
 344             print('%s' % line.rstrip())\r
 345 \r
 346     def execute(self):\r
 347         """\r
 348         Executes the current process.\r
 349 \r
 350         Parameters\r
 351         ----------\r
 352         parameter : type\r
 353             Parameter description.\r
 354 \r
 355         Returns\r
 356         -------\r
 357         type\r
 358              Return value description.\r
 359         """\r
 360 \r
 361         import datetime\r
 362         import traceback\r
 363 \r
 364         try:\r
 365             import subprocess as sp\r
 366         except:\r
 367             sp = None\r
 368 \r
 369         self.start = datetime.datetime.now()\r
 370 \r
 371         cmdargs = [self.cmd]\r
 372         cmdargs.extend(self.args)\r
 373 \r
 374         if self.echo:\r
 375             if sp:\r
 376                 print(\r
 377                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 378             else:\r
 379                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 380 \r
 381         process = None\r
 382         tmp_wrapper = None\r
 383         stdout = None\r
 384         stdin = None\r
 385         parentenv = os.environ\r
 386         parentcwd = os.getcwd()\r
 387 \r
 388         try:\r
 389             # Using *subprocess*.\r
 390             if sp:\r
 391                 if self.batch_wrapper:\r
 392                     cmd = ' '.join(cmdargs)\r
 393                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 394                     write_text(cmd, tmp_wrapper)\r
 395                     print('%s : Running process through wrapper %s\n' % (\r
 396                         self.__class__, tmp_wrapper))\r
 397                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 398                                        stderr=sp.STDOUT,\r
 399                                        cwd=self.cwd, env=self.env)\r
 400                 else:\r
 401                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 402                                        stderr=sp.STDOUT,\r
 403                                        cwd=self.cwd, env=self.env)\r
 404 \r
 405             # using *os.popen4*.\r
 406             else:\r
 407                 if self.env:\r
 408                     os.environ = self.env\r
 409                 if self.cwd:\r
 410                     os.chdir(self.cwd)\r
 411 \r
 412                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 413         except:\r
 414             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 415             traceback.print_exc()\r
 416 \r
 417         # Using *subprocess*\r
 418         if sp:\r
 419             if process is not None:\r
 420                 # pid = process.pid\r
 421                 # log.logLine('process id %s\n' % pid)\r
 422 \r
 423                 try:\r
 424                     # This is more proper python, and resolves some issues with\r
 425                     # a process ending before all of its output has been\r
 426                     # processed, but it also seems to stall when the read\r
 427                     # buffer is near or over its limit. This happens\r
 428                     # relatively frequently with processes that generate lots\r
 429                     # of print statements.\r
 430                     for line in process.stdout:\r
 431                         self.log_line(line)\r
 432 \r
 433                     # So we go with the, um, uglier option below.\r
 434 \r
 435                     # This is now used to ensure that the process has finished.\r
 436                     line = ''\r
 437                     while line is not None and process.poll() is None:\r
 438                         try:\r
 439                             line = process.stdout.readline()\r
 440                         except:\r
 441                             break\r
 442                         # 3.1\r
 443                         try:\r
 444                             # TODO: Investigate previous eroneous statement.\r
 445                             # self.log_line(str(line, encoding='utf-8'))\r
 446                             self.log_line(str(line))\r
 447                         # 2.6\r
 448                         except:\r
 449                             self.log_line(line)\r
 450                 except:\r
 451                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 452 \r
 453                 self.status = process.returncode\r
 454 \r
 455                 if self.batch_wrapper and tmp_wrapper:\r
 456                     try:\r
 457                         os.remove(tmp_wrapper)\r
 458                     except:\r
 459                         print(\r
 460                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 461                         traceback.print_exc()\r
 462 \r
 463         # Using *os.popen4*.\r
 464         else:\r
 465             exit_code = -1\r
 466             try:\r
 467                 stdout_lines = stdout.readlines()\r
 468                 # TODO: Investigate if this is the good behavior, close() does\r
 469                 # not return anything / None.\r
 470                 exit_code = stdout.close()\r
 471 \r
 472                 stdout.close()\r
 473                 stdin.close()\r
 474 \r
 475                 if self.env:\r
 476                     os.environ = parentenv\r
 477                 if self.cwd:\r
 478                     os.chdir(parentcwd)\r
 479 \r
 480                 if len(stdout_lines) > 0:\r
 481                     for line in stdout_lines:\r
 482                         self.log_line(line)\r
 483 \r
 484                 if not exit_code:\r
 485                     exit_code = 0\r
 486             except:\r
 487                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 488 \r
 489             self.status = exit_code\r
 490 \r
 491         self.end = datetime.datetime.now()\r
 492 \r
 493 \r
 494 class ProcessList(Process):\r
 495     """\r
 496     A list of processes with logged output.\r
 497     """\r
 498 \r
 499     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 500         """\r
 501         Object description.\r
 502 \r
 503         Parameters\r
 504         ----------\r
 505         parameter : type\r
 506             Parameter description.\r
 507 \r
 508         Returns\r
 509         -------\r
 510         type\r
 511              Return value description.\r
 512         """\r
 513 \r
 514         Process.__init__(self, description, None, None, cwd, env)\r
 515         'Initialize the standard class variables'\r
 516         self.processes = []\r
 517         self.blocking = blocking\r
 518 \r
 519     def generate_report(self, write_dict):\r
 520         """\r
 521         Generates a log based on the success of the child processes.\r
 522 \r
 523         Parameters\r
 524         ----------\r
 525         parameter : type\r
 526             Parameter description.\r
 527 \r
 528         Returns\r
 529         -------\r
 530         type\r
 531              Return value description.\r
 532         """\r
 533 \r
 534         if self.processes:\r
 535             _status = True\r
 536             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 537 \r
 538             self.log = []\r
 539 \r
 540             for child in self.processes:\r
 541                 if isinstance(child, ProcessList):\r
 542                     child.generate_report(write_dict)\r
 543 \r
 544                 key = child.description\r
 545                 value = child.status\r
 546                 if write_dict['format'] == 'xml':\r
 547                     child_result = (\r
 548                         '%s<result description=\'%s\'>%s</result>' % (\r
 549                             indent, key, value))\r
 550                 else:\r
 551                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 552                 self.log.append(child_result)\r
 553 \r
 554                 if child.status != 0:\r
 555                     _status = False\r
 556             if not _status:\r
 557                 self.status = -1\r
 558             else:\r
 559                 self.status = 0\r
 560         else:\r
 561             self.log = ['No child processes available to generate a report']\r
 562             self.status = -1\r
 563 \r
 564     def write_log_header(self, write_dict):\r
 565         """\r
 566         Object description.\r
 567 \r
 568         Parameters\r
 569         ----------\r
 570         parameter : type\r
 571             Parameter description.\r
 572 \r
 573         Returns\r
 574         -------\r
 575         type\r
 576              Return value description.\r
 577         """\r
 578 \r
 579         self.write_key(write_dict, 'processList', None, 'start')\r
 580         write_dict['indentationLevel'] += 1\r
 581 \r
 582         self.write_key(write_dict, 'description', self.description)\r
 583         self.write_key(write_dict, 'start', self.start)\r
 584         self.write_key(write_dict, 'end', self.end)\r
 585         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 586 \r
 587         self.generate_report(write_dict)\r
 588 \r
 589         self.write_key(write_dict, 'status', self.status)\r
 590 \r
 591     def write_log_footer(self, write_dict):\r
 592         """\r
 593         Object description.\r
 594 \r
 595         Parameters\r
 596         ----------\r
 597         parameter : type\r
 598             Parameter description.\r
 599 \r
 600         Returns\r
 601         -------\r
 602         type\r
 603              Return value description.\r
 604         """\r
 605 \r
 606         write_dict['indentationLevel'] -= 1\r
 607         self.write_key(write_dict, 'processList', None, 'stop')\r
 608 \r
 609     def write_log(self,\r
 610                   log_handle=sys.stdout,\r
 611                   indentation_level=0,\r
 612                   format='xml'):\r
 613         """\r
 614         Writes logging information to the specified handle.\r
 615 \r
 616         Parameters\r
 617         ----------\r
 618         parameter : type\r
 619             Parameter description.\r
 620 \r
 621         Returns\r
 622         -------\r
 623         type\r
 624              Return value description.\r
 625         """\r
 626 \r
 627         write_dict = {}\r
 628         write_dict['logHandle'] = log_handle\r
 629         write_dict['indentationLevel'] = indentation_level\r
 630         write_dict['format'] = format\r
 631 \r
 632         if log_handle:\r
 633             self.write_log_header(write_dict)\r
 634 \r
 635             if self.log:\r
 636                 self.write_key(write_dict, 'output', None, 'start')\r
 637                 for line in self.log:\r
 638                     log_handle.write('%s%s\n' % ('', line))\r
 639                 self.write_key(write_dict, 'output', None, 'stop')\r
 640 \r
 641             if self.processes:\r
 642                 self.write_key(write_dict, 'processes', None, 'start')\r
 643                 for child in self.processes:\r
 644                     child.write_log(log_handle, indentation_level + 1, format)\r
 645                 self.write_key(write_dict, 'processes', None, 'stop')\r
 646 \r
 647             self.write_log_footer(write_dict)\r
 648 \r
 649     def execute(self):\r
 650         """\r
 651         Executes the list of processes.\r
 652 \r
 653         Parameters\r
 654         ----------\r
 655         parameter : type\r
 656             Parameter description.\r
 657 \r
 658         Returns\r
 659         -------\r
 660         type\r
 661              Return value description.\r
 662         """\r
 663 \r
 664         import datetime\r
 665 \r
 666         self.start = datetime.datetime.now()\r
 667 \r
 668         self.status = 0\r
 669         if self.processes:\r
 670             for child in self.processes:\r
 671                 if child:\r
 672                     try:\r
 673                         child.execute()\r
 674                     except:\r
 675                         print('%s : caught exception in child class %s' % (\r
 676                             self.__class__, child.__class__))\r
 677                         traceback.print_exc()\r
 678                         child.status = -1\r
 679 \r
 680                     if self.blocking and child.status != 0:\r
 681                         print('%s : child class %s finished with an error' % (\r
 682                             self.__class__, child.__class__))\r
 683                         self.status = -1\r
 684                         break\r
 685 \r
 686         self.end = datetime.datetime.now()\r
 687 \r
 688 \r
 689 def main():\r
 690     """\r
 691     Object description.\r
 692 \r
 693     Parameters\r
 694     ----------\r
 695     parameter : type\r
 696         Parameter description.\r
 697 \r
 698     Returns\r
 699     -------\r
 700     type\r
 701          Return value description.\r
 702     """\r
 703 \r
 704     import optparse\r
 705 \r
 706     p = optparse.OptionParser(description='A process logging script',\r
 707                               prog='process',\r
 708                               version='process 0.1',\r
 709                               usage=('%prog [options] '\r
 710                                      '[options for the logged process]'))\r
 711     p.add_option('--cmd', '-c', default=None)\r
 712     p.add_option('--log', '-l', default=None)\r
 713 \r
 714     options, arguments = p.parse_args()\r
 715 \r
 716     cmd = options.cmd\r
 717     log_filename = options.log\r
 718 \r
 719     try:\r
 720         args_start = sys.argv.index('--') + 1\r
 721         args = sys.argv[args_start:]\r
 722     except:\r
 723         args_start = len(sys.argv) + 1\r
 724         args = []\r
 725 \r
 726     if cmd is None:\r
 727         print('process: No command specified')\r
 728 \r
 729     # Testing regular logging.\r
 730     process = Process(description='a process', cmd=cmd, args=args)\r
 731 \r
 732     # Testing report generation and writing a log.\r
 733     process_list = ProcessList('a process list')\r
 734     process_list.processes.append(process)\r
 735     process_list.echo = True\r
 736     process_list.execute()\r
 737 \r
 738     process_list.write_log_to_disk(log_filename)\r
 739 \r
 740 \r
 741 if __name__ == '__main__':\r
 742     main()\r