aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['read_text',\r
  21            'write_text',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def read_text(text_file):\r
  28     """\r
  29     Object description.\r
  30 \r
  31     Parameters\r
  32     ----------\r
  33     parameter : type\r
  34         Parameter description.\r
  35 \r
  36     Returns\r
  37     -------\r
  38     type\r
  39          Return value description.\r
  40     """\r
  41 \r
  42     if (text_file != ''):\r
  43         fp = open(text_file, 'rb')\r
  44         # Create a text/plain message\r
  45         text = (fp.read())\r
  46         fp.close()\r
  47     return text\r
  48 \r
  49 \r
  50 def write_text(text, text_file):\r
  51     """\r
  52     Object description.\r
  53 \r
  54     Parameters\r
  55     ----------\r
  56     parameter : type\r
  57         Parameter description.\r
  58 \r
  59     Returns\r
  60     -------\r
  61     type\r
  62          Return value description.\r
  63     """\r
  64 \r
  65     if (text_file != ''):\r
  66         fp = open(text_file, 'wb')\r
  67         # Create a text/plain message\r
  68         fp.write(text)\r
  69         fp.close()\r
  70     return text\r
  71 \r
  72 \r
  73 class Process:\r
  74     """\r
  75     A process with logged output.\r
  76     """\r
  77 \r
  78     def __init__(self,\r
  79                  description=None,\r
  80                  cmd=None,\r
  81                  args=[],\r
  82                  cwd=None,\r
  83                  env=None,\r
  84                  batch_wrapper=False):\r
  85         """\r
  86         Initialize the standard class variables.\r
  87 \r
  88         Parameters\r
  89         ----------\r
  90         parameter : type\r
  91             Parameter description.\r
  92 \r
  93         Returns\r
  94         -------\r
  95         type\r
  96              Return value description.\r
  97         """\r
  98 \r
  99         self.cmd = cmd\r
 100         if not description:\r
 101             self.description = cmd\r
 102         else:\r
 103             self.description = description\r
 104         self.status = None\r
 105         self.args = args\r
 106         self.start = None\r
 107         self.end = None\r
 108         self.log = []\r
 109         self.echo = True\r
 110         self.cwd = cwd\r
 111         self.env = env\r
 112         self.batch_wrapper = batch_wrapper\r
 113         self.process_keys = []\r
 114 \r
 115     def get_elapsed_seconds(self):\r
 116         """\r
 117         Object description.\r
 118 \r
 119         Parameters\r
 120         ----------\r
 121         parameter : type\r
 122             Parameter description.\r
 123 \r
 124         Returns\r
 125         -------\r
 126         type\r
 127              Return value description.\r
 128         """\r
 129 \r
 130         import math\r
 131 \r
 132         if self.end and self.start:\r
 133             delta = (self.end - self.start)\r
 134             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 135                                    int(math.floor(delta.microseconds / 1e3)))\r
 136         else:\r
 137             formatted = None\r
 138         return formatted\r
 139 \r
 140     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 141         """\r
 142         Writes a key / value pair in a supported format.\r
 143 \r
 144         Parameters\r
 145         ----------\r
 146         parameter : type\r
 147             Parameter description.\r
 148 \r
 149         Returns\r
 150         -------\r
 151         type\r
 152              Return value description.\r
 153         """\r
 154 \r
 155         if key != None and (value != None or start_stop != None):\r
 156             indent = '\t' * write_dict['indentationLevel']\r
 157             if write_dict['format'] == 'xml':\r
 158                 if start_stop == 'start':\r
 159                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 160                 elif start_stop == 'stop':\r
 161                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 162                 else:\r
 163                     write_dict['logHandle'].write(\r
 164                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 165             else:  # writeDict['format'] == 'txt':\r
 166                 write_dict['logHandle'].write(\r
 167                     '%s%40s : %s\n' % (indent, key, value))\r
 168 \r
 169     def write_log_header(self, write_dict):\r
 170         """\r
 171         Object description.\r
 172 \r
 173         Parameters\r
 174         ----------\r
 175         parameter : type\r
 176             Parameter description.\r
 177 \r
 178         Returns\r
 179         -------\r
 180         type\r
 181              Return value description.\r
 182         """\r
 183 \r
 184         import platform\r
 185 \r
 186         # Retrieve operating environment information\r
 187         user = None\r
 188         try:\r
 189             user = os.getlogin()\r
 190         except:\r
 191             try:\r
 192                 user = os.getenv('USERNAME')\r
 193                 if user is None:\r
 194                     user = os.getenv('USER')\r
 195             except:\r
 196                 user = 'unknown_user'\r
 197         try:\r
 198             (sysname, nodename, release, version, machine,\r
 199              processor) = platform.uname()\r
 200         except:\r
 201             (sysname, nodename, release, version, machine, processor) = (\r
 202                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 203                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 204         try:\r
 205             hostname = platform.node()\r
 206         except:\r
 207             hostname = 'unknown_hostname'\r
 208 \r
 209         self.write_key(write_dict, 'process', None, 'start')\r
 210         write_dict['indentationLevel'] += 1\r
 211 \r
 212         self.write_key(write_dict, 'description', self.description)\r
 213         self.write_key(write_dict, 'cmd', self.cmd)\r
 214         if self.args:\r
 215             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 216         self.write_key(write_dict, 'start', self.start)\r
 217         self.write_key(write_dict, 'end', self.end)\r
 218         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 219 \r
 220         self.write_key(write_dict, 'user', user)\r
 221         self.write_key(write_dict, 'sysname', sysname)\r
 222         self.write_key(write_dict, 'nodename', nodename)\r
 223         self.write_key(write_dict, 'release', release)\r
 224         self.write_key(write_dict, 'version', version)\r
 225         self.write_key(write_dict, 'machine', machine)\r
 226         self.write_key(write_dict, 'processor', processor)\r
 227 \r
 228         if len(self.process_keys) > 0:\r
 229             self.write_key(write_dict, 'processKeys', None, 'start')\r
 230             for pair in self.process_keys:\r
 231                 (key, value) = pair\r
 232                 write_dict['indentationLevel'] += 1\r
 233                 self.write_key(write_dict, key, value)\r
 234                 write_dict['indentationLevel'] -= 1\r
 235             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 236 \r
 237         self.write_key(write_dict, 'status', self.status)\r
 238 \r
 239     def write_log_footer(self, write_dict):\r
 240         """\r
 241         Object description.\r
 242 \r
 243         Parameters\r
 244         ----------\r
 245         parameter : type\r
 246             Parameter description.\r
 247 \r
 248         Returns\r
 249         -------\r
 250         type\r
 251              Return value description.\r
 252         """\r
 253 \r
 254         write_dict['indentationLevel'] -= 1\r
 255         self.write_key(write_dict, 'process', None, 'stop')\r
 256 \r
 257     def write_log(self,\r
 258                   log_handle=sys.stdout,\r
 259                   indentation_level=0,\r
 260                   format='xml'):\r
 261         """\r
 262         Writes logging information to the specified handle.\r
 263 \r
 264         Parameters\r
 265         ----------\r
 266         parameter : type\r
 267             Parameter description.\r
 268 \r
 269         Returns\r
 270         -------\r
 271         type\r
 272              Return value description.\r
 273         """\r
 274 \r
 275         write_dict = {}\r
 276         write_dict['logHandle'] = log_handle\r
 277         write_dict['indentationLevel'] = indentation_level\r
 278         write_dict['format'] = format\r
 279 \r
 280         if log_handle:\r
 281             self.write_log_header(write_dict)\r
 282 \r
 283             if self.log:\r
 284                 self.write_key(write_dict, 'output', None, 'start')\r
 285                 if format == 'xml':\r
 286                     log_handle.write('<![CDATA[\n')\r
 287                 for line in self.log:\r
 288                     log_handle.write('%s%s\n' % ('', line))\r
 289                 if format == 'xml':\r
 290                     log_handle.write(']]>\n')\r
 291                 self.write_key(write_dict, 'output', None, 'stop')\r
 292 \r
 293             self.write_log_footer(write_dict)\r
 294 \r
 295     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 296         """\r
 297         Object description.\r
 298 \r
 299         Parameters\r
 300         ----------\r
 301         parameter : type\r
 302             Parameter description.\r
 303 \r
 304         Returns\r
 305         -------\r
 306         type\r
 307              Return value description.\r
 308         """\r
 309 \r
 310         if log_filename:\r
 311             try:\r
 312                 # This also doesn't seem like the best structure...\r
 313                 # 3.1\r
 314                 try:\r
 315                     log_handle = open(log_filename,\r
 316                                       mode='wt',\r
 317                                       encoding='utf-8')\r
 318                 # 2.6\r
 319                 except:\r
 320                     log_handle = open(log_filename,\r
 321                                       mode='wt')\r
 322             except:\r
 323                 print('Couldn\'t open log : %s' % log_filename)\r
 324                 log_handle = None\r
 325 \r
 326         if log_handle:\r
 327             if header:\r
 328                 if format == 'xml':\r
 329                     log_handle.write('<![CDATA[\n')\r
 330                 log_handle.write(header)\r
 331                 if format == 'xml':\r
 332                     log_handle.write(']]>\n')\r
 333             self.write_log(log_handle)\r
 334             log_handle.close()\r
 335 \r
 336     def log_line(self, line):\r
 337         """\r
 338         Adds a line of text to the log.\r
 339 \r
 340         Parameters\r
 341         ----------\r
 342         parameter : type\r
 343             Parameter description.\r
 344 \r
 345         Returns\r
 346         -------\r
 347         type\r
 348              Return value description.\r
 349         """\r
 350 \r
 351         self.log.append(line.rstrip())\r
 352         if self.echo:\r
 353             print('%s' % line.rstrip())\r
 354 \r
 355     def execute(self):\r
 356         """\r
 357         Executes the current process.\r
 358 \r
 359         Parameters\r
 360         ----------\r
 361         parameter : type\r
 362             Parameter description.\r
 363 \r
 364         Returns\r
 365         -------\r
 366         type\r
 367              Return value description.\r
 368         """\r
 369 \r
 370         import datetime\r
 371         import traceback\r
 372 \r
 373         try:\r
 374             import subprocess as sp\r
 375         except:\r
 376             sp = None\r
 377 \r
 378         self.start = datetime.datetime.now()\r
 379 \r
 380         cmdargs = [self.cmd]\r
 381         cmdargs.extend(self.args)\r
 382 \r
 383         if self.echo:\r
 384             if sp:\r
 385                 print(\r
 386                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 387             else:\r
 388                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 389 \r
 390         # intialize a few variables that may or may not be set later\r
 391         process = None\r
 392         tmp_wrapper = None\r
 393         stdout = None\r
 394         stdin = None\r
 395         parentenv = os.environ\r
 396         parentcwd = os.getcwd()\r
 397 \r
 398         try:\r
 399             # Using subprocess\r
 400             if sp:\r
 401                 if self.batch_wrapper:\r
 402                     cmd = ' '.join(cmdargs)\r
 403                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 404                     write_text(cmd, tmp_wrapper)\r
 405                     print('%s : Running process through wrapper %s\n' % (\r
 406                         self.__class__, tmp_wrapper))\r
 407                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 408                                        stderr=sp.STDOUT,\r
 409                                        cwd=self.cwd, env=self.env)\r
 410                 else:\r
 411                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 412                                        stderr=sp.STDOUT,\r
 413                                        cwd=self.cwd, env=self.env)\r
 414 \r
 415             # using os.popen4\r
 416             else:\r
 417                 if self.env:\r
 418                     os.environ = self.env\r
 419                 if self.cwd:\r
 420                     os.chdir(self.cwd)\r
 421 \r
 422                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 423         except:\r
 424             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 425             traceback.print_exc()\r
 426 \r
 427         # Using subprocess\r
 428         if sp:\r
 429             if process != None:\r
 430                 # pid = process.pid\r
 431                 # log.logLine('process id %s\n' % pid)\r
 432 \r
 433                 try:\r
 434                     # This is more proper python, and resolves some issues with\r
 435                     # a process ending before all of its output has been\r
 436                     # processed, but it also seems to stall when the read\r
 437                     # buffer is near or over it's limit. this happens\r
 438                     # relatively frequently with processes that generate lots\r
 439                     # of print statements.\r
 440                     for line in process.stdout:\r
 441                         self.log_line(line)\r
 442                     #\r
 443                     # So we go with the, um, uglier  option below\r
 444 \r
 445                     # This is now used to ensure that the process has finished\r
 446                     line = ''\r
 447                     while line != None and process.poll() is None:\r
 448                         try:\r
 449                             line = process.stdout.readline()\r
 450                         except:\r
 451                             break\r
 452                         # 3.1\r
 453                         try:\r
 454                             self.log_line(str(line, encoding='utf-8'))\r
 455                         # 2.6\r
 456                         except:\r
 457                             self.log_line(line)\r
 458                 except:\r
 459                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 460 \r
 461                 self.status = process.returncode\r
 462 \r
 463                 if self.batch_wrapper and tmp_wrapper:\r
 464                     try:\r
 465                         os.remove(tmp_wrapper)\r
 466                     except:\r
 467                         print(\r
 468                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 469                         traceback.print_exc()\r
 470 \r
 471         # Using os.popen4\r
 472         else:\r
 473             exit_code = -1\r
 474             try:\r
 475                 # print('reading stdout lines')\r
 476                 stdout_lines = stdout.readlines()\r
 477                 exit_code = stdout.close()\r
 478 \r
 479                 stdout.close()\r
 480                 stdin.close()\r
 481 \r
 482                 if self.env:\r
 483                     os.environ = parentenv\r
 484                 if self.cwd:\r
 485                     os.chdir(parentcwd)\r
 486 \r
 487                 if len(stdout_lines) > 0:\r
 488                     for line in stdout_lines:\r
 489                         self.log_line(line)\r
 490 \r
 491                 if not exit_code:\r
 492                     exit_code = 0\r
 493             except:\r
 494                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 495 \r
 496             self.status = exit_code\r
 497 \r
 498         self.end = datetime.datetime.now()\r
 499 \r
 500 \r
 501 class ProcessList(Process):\r
 502     """\r
 503     A list of processes with logged output.\r
 504     """\r
 505 \r
 506     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 507         """\r
 508         Object description.\r
 509 \r
 510         Parameters\r
 511         ----------\r
 512         parameter : type\r
 513             Parameter description.\r
 514 \r
 515         Returns\r
 516         -------\r
 517         type\r
 518              Return value description.\r
 519         """\r
 520 \r
 521         Process.__init__(self, description, None, None, cwd, env)\r
 522         'Initialize the standard class variables'\r
 523         self.processes = []\r
 524         self.blocking = blocking\r
 525 \r
 526     def generate_report(self, write_dict):\r
 527         """\r
 528         Generates a log based on the success of the child processes.\r
 529 \r
 530         Parameters\r
 531         ----------\r
 532         parameter : type\r
 533             Parameter description.\r
 534 \r
 535         Returns\r
 536         -------\r
 537         type\r
 538              Return value description.\r
 539         """\r
 540 \r
 541         if self.processes:\r
 542             _status = True\r
 543             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 544 \r
 545             self.log = []\r
 546 \r
 547             for child in self.processes:\r
 548                 if isinstance(child, ProcessList):\r
 549                     child.generate_report(write_dict)\r
 550 \r
 551                 child_result = ''\r
 552                 key = child.description\r
 553                 value = child.status\r
 554                 if write_dict['format'] == 'xml':\r
 555                     child_result = (\r
 556                         '%s<result description=\'%s\'>%s</result>' % (\r
 557                             indent, key, value))\r
 558                 else:  # writeDict['format'] == 'txt':\r
 559                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 560                 self.log.append(child_result)\r
 561 \r
 562                 if child.status != 0:\r
 563                     _status = False\r
 564             if not _status:\r
 565                 self.status = -1\r
 566             else:\r
 567                 self.status = 0\r
 568         else:\r
 569             self.log = ['No child processes available to generate a report']\r
 570             self.status = -1\r
 571 \r
 572     def write_log_header(self, write_dict):\r
 573         """\r
 574         Object description.\r
 575 \r
 576         Parameters\r
 577         ----------\r
 578         parameter : type\r
 579             Parameter description.\r
 580 \r
 581         Returns\r
 582         -------\r
 583         type\r
 584              Return value description.\r
 585         """\r
 586 \r
 587         self.write_key(write_dict, 'processList', None, 'start')\r
 588         write_dict['indentationLevel'] += 1\r
 589 \r
 590         self.write_key(write_dict, 'description', self.description)\r
 591         self.write_key(write_dict, 'start', self.start)\r
 592         self.write_key(write_dict, 'end', self.end)\r
 593         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 594 \r
 595         self.generate_report(write_dict)\r
 596 \r
 597         self.write_key(write_dict, 'status', self.status)\r
 598 \r
 599     def write_log_footer(self, write_dict):\r
 600         """\r
 601         Object description.\r
 602 \r
 603         Parameters\r
 604         ----------\r
 605         parameter : type\r
 606             Parameter description.\r
 607 \r
 608         Returns\r
 609         -------\r
 610         type\r
 611              Return value description.\r
 612         """\r
 613 \r
 614         write_dict['indentationLevel'] -= 1\r
 615         self.write_key(write_dict, 'processList', None, 'stop')\r
 616 \r
 617     def write_log(self,\r
 618                   log_handle=sys.stdout,\r
 619                   indentation_level=0,\r
 620                   format='xml'):\r
 621         """\r
 622         Writes logging information to the specified handle.\r
 623 \r
 624         Parameters\r
 625         ----------\r
 626         parameter : type\r
 627             Parameter description.\r
 628 \r
 629         Returns\r
 630         -------\r
 631         type\r
 632              Return value description.\r
 633         """\r
 634 \r
 635         write_dict = {}\r
 636         write_dict['logHandle'] = log_handle\r
 637         write_dict['indentationLevel'] = indentation_level\r
 638         write_dict['format'] = format\r
 639 \r
 640         if log_handle:\r
 641             self.write_log_header(write_dict)\r
 642 \r
 643             if self.log:\r
 644                 self.write_key(write_dict, 'output', None, 'start')\r
 645                 for line in self.log:\r
 646                     log_handle.write('%s%s\n' % ('', line))\r
 647                 self.write_key(write_dict, 'output', None, 'stop')\r
 648 \r
 649             if self.processes:\r
 650                 self.write_key(write_dict, 'processes', None, 'start')\r
 651                 for child in self.processes:\r
 652                     child.write_log(log_handle, indentation_level + 1, format)\r
 653                 self.write_key(write_dict, 'processes', None, 'stop')\r
 654 \r
 655             self.write_log_footer(write_dict)\r
 656 \r
 657     def execute(self):\r
 658         """\r
 659         Executes the list of processes.\r
 660 \r
 661         Parameters\r
 662         ----------\r
 663         parameter : type\r
 664             Parameter description.\r
 665 \r
 666         Returns\r
 667         -------\r
 668         type\r
 669              Return value description.\r
 670         """\r
 671 \r
 672         import datetime\r
 673 \r
 674         self.start = datetime.datetime.now()\r
 675 \r
 676         self.status = 0\r
 677         if self.processes:\r
 678             for child in self.processes:\r
 679                 if child:\r
 680                     try:\r
 681                         child.execute()\r
 682                     except:\r
 683                         print('%s : caught exception in child class %s' % (\r
 684                             self.__class__, child.__class__))\r
 685                         traceback.print_exc()\r
 686                         child.status = -1\r
 687 \r
 688                     if self.blocking and child.status != 0:\r
 689                         print('%s : child class %s finished with an error' % (\r
 690                             self.__class__, child.__class__))\r
 691                         self.status = -1\r
 692                         break\r
 693 \r
 694         self.end = datetime.datetime.now()\r
 695 \r
 696 \r
 697 def main():\r
 698     """\r
 699     Object description.\r
 700 \r
 701     Parameters\r
 702     ----------\r
 703     parameter : type\r
 704         Parameter description.\r
 705 \r
 706     Returns\r
 707     -------\r
 708     type\r
 709          Return value description.\r
 710     """\r
 711 \r
 712     import optparse\r
 713 \r
 714     p = optparse.OptionParser(description='A process logging script',\r
 715                               prog='process',\r
 716                               version='process 0.1',\r
 717                               usage=('%prog [options] '\r
 718                                      '[options for the logged process]'))\r
 719     p.add_option('--cmd', '-c', default=None)\r
 720     p.add_option('--log', '-l', default=None)\r
 721 \r
 722     options, arguments = p.parse_args()\r
 723 \r
 724     #\r
 725     # Get options\r
 726     # \r
 727     cmd = options.cmd\r
 728     log_filename = options.log\r
 729 \r
 730     try:\r
 731         args_start = sys.argv.index('--') + 1\r
 732         args = sys.argv[args_start:]\r
 733     except:\r
 734         args_start = len(sys.argv) + 1\r
 735         args = []\r
 736 \r
 737     if cmd is None:\r
 738         print('process: No command specified')\r
 739 \r
 740     #\r
 741     # Test regular logging\r
 742     #\r
 743     process = Process(description='a process', cmd=cmd, args=args)\r
 744 \r
 745     #\r
 746     # Test report generation and writing a log\r
 747     #\r
 748     process_list = ProcessList('a process list')\r
 749     process_list.processes.append(process)\r
 750     process_list.echo = True\r
 751     process_list.execute()\r
 752 \r
 753     process_list.write_log_to_disk(log_filename)\r
 754 \r
 755 \r
 756 if __name__ == '__main__':\r
 757     main()\r