aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status\r
   6 of a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['readText',\r
  21            'writeText',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def readText(textFile):\r
  28     """\r
  29     Object description.\r
  30 \r
  31     Parameters\r
  32     ----------\r
  33     parameter : type\r
  34         Parameter description.\r
  35 \r
  36     Returns\r
  37     -------\r
  38     type\r
  39          Return value description.\r
  40     """\r
  41 \r
  42     if (textFile != ""):\r
  43         fp = open(textFile, 'rb')\r
  44         # Create a text/plain message\r
  45         text = (fp.read())\r
  46         fp.close()\r
  47     return text\r
  48 \r
  49 \r
  50 def writeText(text, textFile):\r
  51     """\r
  52     Object description.\r
  53 \r
  54     Parameters\r
  55     ----------\r
  56     parameter : type\r
  57         Parameter description.\r
  58 \r
  59     Returns\r
  60     -------\r
  61     type\r
  62          Return value description.\r
  63     """\r
  64 \r
  65     if (textFile != ""):\r
  66         fp = open(textFile, 'wb')\r
  67         # Create a text/plain message\r
  68         fp.write(text)\r
  69         fp.close()\r
  70     return text\r
  71 \r
  72 \r
  73 class Process:\r
  74     """\r
  75     A process with logged output.\r
  76     """\r
  77 \r
  78     def __init__(self,\r
  79                  description=None,\r
  80                  cmd=None,\r
  81                  args=[],\r
  82                  cwd=None,\r
  83                  env=None,\r
  84                  batchWrapper=False):\r
  85         """\r
  86         Initialize the standard class variables.\r
  87 \r
  88         Parameters\r
  89         ----------\r
  90         parameter : type\r
  91             Parameter description.\r
  92 \r
  93         Returns\r
  94         -------\r
  95         type\r
  96              Return value description.\r
  97         """\r
  98 \r
  99         self.cmd = cmd\r
 100         if not description:\r
 101             self.description = cmd\r
 102         else:\r
 103             self.description = description\r
 104         self.status = None\r
 105         self.args = args\r
 106         self.start = None\r
 107         self.end = None\r
 108         self.log = []\r
 109         self.echo = True\r
 110         self.cwd = cwd\r
 111         self.env = env\r
 112         self.batchWrapper = batchWrapper\r
 113         self.processKeys = []\r
 114 \r
 115     def getElapsedSeconds(self):\r
 116         """\r
 117         Object description.\r
 118 \r
 119         Parameters\r
 120         ----------\r
 121         parameter : type\r
 122             Parameter description.\r
 123 \r
 124         Returns\r
 125         -------\r
 126         type\r
 127              Return value description.\r
 128         """\r
 129 \r
 130         import math\r
 131 \r
 132         if self.end and self.start:\r
 133             delta = (self.end - self.start)\r
 134             formatted = "%s.%s" % (delta.days * 86400 + delta.seconds,\r
 135                                    int(math.floor(delta.microseconds / 1e3)))\r
 136         else:\r
 137             formatted = None\r
 138         return formatted\r
 139 \r
 140     def writeKey(self, writeDict, key=None, value=None, startStop=None):\r
 141         """\r
 142         Writes a key / value pair in a supported format.\r
 143 \r
 144         Parameters\r
 145         ----------\r
 146         parameter : type\r
 147             Parameter description.\r
 148 \r
 149         Returns\r
 150         -------\r
 151         type\r
 152              Return value description.\r
 153         """\r
 154 \r
 155         if key != None and (value != None or startStop != None):\r
 156             indent = '\t' * writeDict['indentationLevel']\r
 157             if writeDict['format'] == 'xml':\r
 158                 if startStop == 'start':\r
 159                     writeDict['logHandle'].write("%s<%s>\n" % (indent, key))\r
 160                 elif startStop == 'stop':\r
 161                     writeDict['logHandle'].write("%s</%s>\n" % (indent, key))\r
 162                 else:\r
 163                     writeDict['logHandle'].write(\r
 164                         "%s<%s>%s</%s>\n" % (indent, key, value, key))\r
 165             else:  # writeDict['format'] == 'txt':\r
 166                 writeDict['logHandle'].write(\r
 167                     "%s%40s : %s\n" % (indent, key, value))\r
 168 \r
 169     def writeLogHeader(self, writeDict):\r
 170         """\r
 171         Object description.\r
 172 \r
 173         Parameters\r
 174         ----------\r
 175         parameter : type\r
 176             Parameter description.\r
 177 \r
 178         Returns\r
 179         -------\r
 180         type\r
 181              Return value description.\r
 182         """\r
 183 \r
 184         import platform\r
 185 \r
 186         # Retrieve operating environment information\r
 187         user = None\r
 188         try:\r
 189             user = os.getlogin()\r
 190         except:\r
 191             try:\r
 192                 user = os.getenv("USERNAME")\r
 193                 if user == None:\r
 194                     user = os.getenv("USER")\r
 195             except:\r
 196                 user = "unknown_user"\r
 197         try:\r
 198             (sysname, nodename, release, version, machine,\r
 199              processor) = platform.uname()\r
 200         except:\r
 201             (sysname, nodename, release, version, machine, processor) = (\r
 202                 "unknown_sysname", "unknown_nodename", "unknown_release",\r
 203                 "unknown_version", "unknown_machine", "unknown_processor")\r
 204         try:\r
 205             hostname = platform.node()\r
 206         except:\r
 207             hostname = "unknown_hostname"\r
 208 \r
 209         self.writeKey(writeDict, 'process', None, 'start')\r
 210         writeDict['indentationLevel'] += 1\r
 211 \r
 212         self.writeKey(writeDict, 'description', self.description)\r
 213         self.writeKey(writeDict, 'cmd', self.cmd)\r
 214         if self.args:\r
 215             self.writeKey(writeDict, 'args', ' '.join(self.args))\r
 216         self.writeKey(writeDict, 'start', self.start)\r
 217         self.writeKey(writeDict, 'end', self.end)\r
 218         self.writeKey(writeDict, 'elapsed', self.getElapsedSeconds())\r
 219 \r
 220         self.writeKey(writeDict, 'user', user)\r
 221         self.writeKey(writeDict, 'sysname', sysname)\r
 222         self.writeKey(writeDict, 'nodename', nodename)\r
 223         self.writeKey(writeDict, 'release', release)\r
 224         self.writeKey(writeDict, 'version', version)\r
 225         self.writeKey(writeDict, 'machine', machine)\r
 226         self.writeKey(writeDict, 'processor', processor)\r
 227 \r
 228         if len(self.processKeys) > 0:\r
 229             self.writeKey(writeDict, 'processKeys', None, 'start')\r
 230             for pair in self.processKeys:\r
 231                 (key, value) = pair\r
 232                 writeDict['indentationLevel'] += 1\r
 233                 self.writeKey(writeDict, key, value)\r
 234                 writeDict['indentationLevel'] -= 1\r
 235             self.writeKey(writeDict, 'processKeys', None, 'stop')\r
 236 \r
 237         self.writeKey(writeDict, 'status', self.status)\r
 238 \r
 239     def writeLogFooter(self, writeDict):\r
 240         """\r
 241         Object description.\r
 242 \r
 243         Parameters\r
 244         ----------\r
 245         parameter : type\r
 246             Parameter description.\r
 247 \r
 248         Returns\r
 249         -------\r
 250         type\r
 251              Return value description.\r
 252         """\r
 253 \r
 254         writeDict['indentationLevel'] -= 1\r
 255         self.writeKey(writeDict, 'process', None, 'stop')\r
 256 \r
 257     def writeLog(self, logHandle=sys.stdout, indentationLevel=0, format='xml'):\r
 258         """\r
 259         Writes logging information to the specified handle.\r
 260 \r
 261         Parameters\r
 262         ----------\r
 263         parameter : type\r
 264             Parameter description.\r
 265 \r
 266         Returns\r
 267         -------\r
 268         type\r
 269              Return value description.\r
 270         """\r
 271 \r
 272         writeDict = {}\r
 273         writeDict['logHandle'] = logHandle\r
 274         writeDict['indentationLevel'] = indentationLevel\r
 275         writeDict['format'] = format\r
 276 \r
 277         if logHandle:\r
 278             self.writeLogHeader(writeDict)\r
 279 \r
 280             if self.log:\r
 281                 self.writeKey(writeDict, 'output', None, 'start')\r
 282                 if format == 'xml':\r
 283                     logHandle.write("<![CDATA[\n")\r
 284                 for line in self.log:\r
 285                     logHandle.write('%s%s\n' % ("", line))\r
 286                 if format == 'xml':\r
 287                     logHandle.write("]]>\n")\r
 288                 self.writeKey(writeDict, 'output', None, 'stop')\r
 289 \r
 290             self.writeLogFooter(writeDict)\r
 291 \r
 292     def writeLogToDisk(self, logFilename=None, format='xml', header=None):\r
 293         """\r
 294         Object description.\r
 295 \r
 296         Parameters\r
 297         ----------\r
 298         parameter : type\r
 299             Parameter description.\r
 300 \r
 301         Returns\r
 302         -------\r
 303         type\r
 304              Return value description.\r
 305         """\r
 306 \r
 307         if logFilename:\r
 308             try:\r
 309                 # This also doesn't seem like the best structure...\r
 310                 # 3.1\r
 311                 try:\r
 312                     logHandle = open(logFilename, mode='wt', encoding="utf-8")\r
 313                 # 2.6\r
 314                 except:\r
 315                     logHandle = open(logFilename, mode='wt')\r
 316             except:\r
 317                 print("Couldn't open log : %s" % logFilename)\r
 318                 logHandle = None\r
 319 \r
 320         if logHandle:\r
 321             if header:\r
 322                 if format == 'xml':\r
 323                     logHandle.write("<![CDATA[\n")\r
 324                 logHandle.write(header)\r
 325                 if format == 'xml':\r
 326                     logHandle.write("]]>\n")\r
 327             self.writeLog(logHandle)\r
 328             logHandle.close()\r
 329 \r
 330     def logLine(self, line):\r
 331         """\r
 332         Adds a line of text to the log.\r
 333 \r
 334         Parameters\r
 335         ----------\r
 336         parameter : type\r
 337             Parameter description.\r
 338 \r
 339         Returns\r
 340         -------\r
 341         type\r
 342              Return value description.\r
 343         """\r
 344 \r
 345         self.log.append(line.rstrip())\r
 346         if self.echo:\r
 347             print("%s" % line.rstrip())\r
 348 \r
 349     def execute(self):\r
 350         """\r
 351         Executes the current process.\r
 352 \r
 353         Parameters\r
 354         ----------\r
 355         parameter : type\r
 356             Parameter description.\r
 357 \r
 358         Returns\r
 359         -------\r
 360         type\r
 361              Return value description.\r
 362         """\r
 363 \r
 364         import datetime\r
 365         import traceback\r
 366 \r
 367         try:\r
 368             import subprocess as sp\r
 369         except:\r
 370             sp = None\r
 371 \r
 372         self.start = datetime.datetime.now()\r
 373 \r
 374         cmdargs = [self.cmd]\r
 375         cmdargs.extend(self.args)\r
 376 \r
 377         if self.echo:\r
 378             if sp:\r
 379                 print(\r
 380                     "\n%s : %s\n" % (self.__class__, sp.list2cmdline(cmdargs)))\r
 381             else:\r
 382                 print("\n%s : %s\n" % (self.__class__, " ".join(cmdargs)))\r
 383 \r
 384         # intialize a few variables that may or may not be set later\r
 385         process = None\r
 386         tmpWrapper = None\r
 387         stdout = None\r
 388         stdin = None\r
 389         parentenv = os.environ\r
 390         parentcwd = os.getcwd()\r
 391 \r
 392         try:\r
 393             # Using subprocess\r
 394             if sp:\r
 395                 if self.batchWrapper:\r
 396                     cmd = " ".join(cmdargs)\r
 397                     tmpWrapper = os.path.join(self.cwd, "process.bat")\r
 398                     writeText(cmd, tmpWrapper)\r
 399                     print("%s : Running process through wrapper %s\n" % (\r
 400                         self.__class__, tmpWrapper))\r
 401                     process = sp.Popen([tmpWrapper], stdout=sp.PIPE,\r
 402                                        stderr=sp.STDOUT,\r
 403                                        cwd=self.cwd, env=self.env)\r
 404                 else:\r
 405                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 406                                        stderr=sp.STDOUT,\r
 407                                        cwd=self.cwd, env=self.env)\r
 408 \r
 409             # using os.popen4\r
 410             else:\r
 411                 if self.env:\r
 412                     os.environ = self.env\r
 413                 if self.cwd:\r
 414                     os.chdir(self.cwd)\r
 415 \r
 416                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 417         except:\r
 418             print("Couldn't execute command : %s" % cmdargs[0])\r
 419             traceback.print_exc()\r
 420 \r
 421         # Using subprocess\r
 422         if sp:\r
 423             if process != None:\r
 424                 # pid = process.pid\r
 425                 # log.logLine("process id %s\n" % pid)\r
 426 \r
 427                 try:\r
 428                     # This is more proper python, and resolves some issues with\r
 429                     # a process ending before all of its output has been\r
 430                     # processed, but it also seems to stall when the read\r
 431                     # buffer is near or over it's limit. this happens\r
 432                     # relatively frequently with processes that generate lots\r
 433                     # of print statements.\r
 434                     for line in process.stdout:\r
 435                         self.logLine(line)\r
 436                     #\r
 437                     # So we go with the, um, uglier  option below\r
 438 \r
 439                     # This is now used to ensure that the process has finished\r
 440                     line = ""\r
 441                     while line != None and process.poll() == None:\r
 442                         try:\r
 443                             line = process.stdout.readline()\r
 444                         except:\r
 445                             break\r
 446                         # 3.1\r
 447                         try:\r
 448                             self.logLine(str(line, encoding="utf-8"))\r
 449                         # 2.6\r
 450                         except:\r
 451                             self.logLine(line)\r
 452                 except:\r
 453                     self.logLine("Logging error : %s" % sys.exc_info()[0])\r
 454 \r
 455                 self.status = process.returncode\r
 456 \r
 457                 if self.batchWrapper and tmpWrapper:\r
 458                     try:\r
 459                         os.remove(tmpWrapper)\r
 460                     except:\r
 461                         print("Couldn't remove temp wrapper : %s" % tmpWrapper)\r
 462                         traceback.print_exc()\r
 463 \r
 464         # Using os.popen4\r
 465         else:\r
 466             exitCode = -1\r
 467             try:\r
 468                 # print("reading stdout lines")\r
 469                 stdoutLines = stdout.readlines()\r
 470                 exitCode = stdout.close()\r
 471 \r
 472                 stdout.close()\r
 473                 stdin.close()\r
 474 \r
 475                 if self.env:\r
 476                     os.environ = parentenv\r
 477                 if self.cwd:\r
 478                     os.chdir(parentcwd)\r
 479 \r
 480                 if len(stdoutLines) > 0:\r
 481                     for line in stdoutLines:\r
 482                         self.logLine(line)\r
 483 \r
 484                 if not exitCode:\r
 485                     exitCode = 0\r
 486             except:\r
 487                 self.logLine("Logging error : %s" % sys.exc_info()[0])\r
 488 \r
 489             self.status = exitCode\r
 490 \r
 491         self.end = datetime.datetime.now()\r
 492 \r
 493 \r
 494 class ProcessList(Process):\r
 495     """\r
 496     A list of processes with logged output.\r
 497     """\r
 498 \r
 499     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 500         """\r
 501         Object description.\r
 502 \r
 503         Parameters\r
 504         ----------\r
 505         parameter : type\r
 506             Parameter description.\r
 507 \r
 508         Returns\r
 509         -------\r
 510         type\r
 511              Return value description.\r
 512         """\r
 513 \r
 514         Process.__init__(self, description, None, None, cwd, env)\r
 515         "Initialize the standard class variables"\r
 516         self.processes = []\r
 517         self.blocking = blocking\r
 518 \r
 519     def generateReport(self, writeDict):\r
 520         """\r
 521         Generates a log based on the success of the child processes.\r
 522 \r
 523         Parameters\r
 524         ----------\r
 525         parameter : type\r
 526             Parameter description.\r
 527 \r
 528         Returns\r
 529         -------\r
 530         type\r
 531              Return value description.\r
 532         """\r
 533 \r
 534         if self.processes:\r
 535             _status = True\r
 536             indent = '\t' * (writeDict['indentationLevel'] + 1)\r
 537 \r
 538             self.log = []\r
 539 \r
 540             for child in self.processes:\r
 541                 if isinstance(child, ProcessList):\r
 542                     child.generateReport(writeDict)\r
 543 \r
 544                 childResult = ""\r
 545                 key = child.description\r
 546                 value = child.status\r
 547                 if writeDict['format'] == 'xml':\r
 548                     childResult = (\r
 549                         "%s<result description=\"%s\">%s</result>" % (\r
 550                             indent, key, value))\r
 551                 else:  # writeDict['format'] == 'txt':\r
 552                     childResult = ("%s%40s : %s" % (indent, key, value))\r
 553                 self.log.append(childResult)\r
 554 \r
 555                 if child.status != 0:\r
 556                     _status = False\r
 557             if not _status:\r
 558                 self.status = -1\r
 559             else:\r
 560                 self.status = 0\r
 561         else:\r
 562             self.log = ["No child processes available to generate a report"]\r
 563             self.status = -1\r
 564 \r
 565     def writeLogHeader(self, writeDict):\r
 566         """\r
 567         Object description.\r
 568 \r
 569         Parameters\r
 570         ----------\r
 571         parameter : type\r
 572             Parameter description.\r
 573 \r
 574         Returns\r
 575         -------\r
 576         type\r
 577              Return value description.\r
 578         """\r
 579 \r
 580         self.writeKey(writeDict, 'processList', None, 'start')\r
 581         writeDict['indentationLevel'] += 1\r
 582 \r
 583         self.writeKey(writeDict, 'description', self.description)\r
 584         self.writeKey(writeDict, 'start', self.start)\r
 585         self.writeKey(writeDict, 'end', self.end)\r
 586         self.writeKey(writeDict, 'elapsed', self.getElapsedSeconds())\r
 587 \r
 588         self.generateReport(writeDict)\r
 589 \r
 590         self.writeKey(writeDict, 'status', self.status)\r
 591 \r
 592     def writeLogFooter(self, writeDict):\r
 593         """\r
 594         Object description.\r
 595 \r
 596         Parameters\r
 597         ----------\r
 598         parameter : type\r
 599             Parameter description.\r
 600 \r
 601         Returns\r
 602         -------\r
 603         type\r
 604              Return value description.\r
 605         """\r
 606 \r
 607         writeDict['indentationLevel'] -= 1\r
 608         self.writeKey(writeDict, 'processList', None, 'stop')\r
 609 \r
 610     def writeLog(self, logHandle=sys.stdout, indentationLevel=0, format='xml'):\r
 611         """\r
 612         Writes logging information to the specified handle.\r
 613 \r
 614         Parameters\r
 615         ----------\r
 616         parameter : type\r
 617             Parameter description.\r
 618 \r
 619         Returns\r
 620         -------\r
 621         type\r
 622              Return value description.\r
 623         """\r
 624 \r
 625         writeDict = {}\r
 626         writeDict['logHandle'] = logHandle\r
 627         writeDict['indentationLevel'] = indentationLevel\r
 628         writeDict['format'] = format\r
 629 \r
 630         if logHandle:\r
 631             self.writeLogHeader(writeDict)\r
 632 \r
 633             if self.log:\r
 634                 self.writeKey(writeDict, 'output', None, 'start')\r
 635                 for line in self.log:\r
 636                     logHandle.write('%s%s\n' % ("", line))\r
 637                 self.writeKey(writeDict, 'output', None, 'stop')\r
 638 \r
 639             if self.processes:\r
 640                 self.writeKey(writeDict, 'processes', None, 'start')\r
 641                 for child in self.processes:\r
 642                     child.writeLog(logHandle, indentationLevel + 1, format)\r
 643                 self.writeKey(writeDict, 'processes', None, 'stop')\r
 644 \r
 645             self.writeLogFooter(writeDict)\r
 646 \r
 647     def execute(self):\r
 648         """\r
 649         Executes the list of processes.\r
 650 \r
 651         Parameters\r
 652         ----------\r
 653         parameter : type\r
 654             Parameter description.\r
 655 \r
 656         Returns\r
 657         -------\r
 658         type\r
 659              Return value description.\r
 660         """\r
 661 \r
 662         import datetime\r
 663 \r
 664         self.start = datetime.datetime.now()\r
 665 \r
 666         self.status = 0\r
 667         if self.processes:\r
 668             for child in self.processes:\r
 669                 if child:\r
 670                     try:\r
 671                         child.execute()\r
 672                     except:\r
 673                         print("%s : caught exception in child class %s" % (\r
 674                             self.__class__, child.__class__))\r
 675                         traceback.print_exc()\r
 676                         child.status = -1\r
 677 \r
 678                     if self.blocking and child.status != 0:\r
 679                         print("%s : child class %s finished with an error" % (\r
 680                             self.__class__, child.__class__))\r
 681                         self.status = -1\r
 682                         break\r
 683 \r
 684         self.end = datetime.datetime.now()\r
 685 \r
 686 \r
 687 def main():\r
 688     """\r
 689     Object description.\r
 690 \r
 691     Parameters\r
 692     ----------\r
 693     parameter : type\r
 694         Parameter description.\r
 695 \r
 696     Returns\r
 697     -------\r
 698     type\r
 699          Return value description.\r
 700     """\r
 701 \r
 702     import optparse\r
 703 \r
 704     p = optparse.OptionParser(description='A process logging script',\r
 705                               prog='process',\r
 706                               version='process 0.1',\r
 707                               usage=('%prog [options] '\r
 708                                      '[options for the logged process]'))\r
 709     p.add_option('--cmd', '-c', default=None)\r
 710     p.add_option('--log', '-l', default=None)\r
 711 \r
 712     options, arguments = p.parse_args()\r
 713 \r
 714     #\r
 715     # Get options\r
 716     # \r
 717     cmd = options.cmd\r
 718     logFilename = options.log\r
 719 \r
 720     try:\r
 721         argsStart = sys.argv.index('--') + 1\r
 722         args = sys.argv[argsStart:]\r
 723     except:\r
 724         argsStart = len(sys.argv) + 1\r
 725         args = []\r
 726 \r
 727     if cmd == None:\r
 728         print("process: No command specified")\r
 729 \r
 730     #\r
 731     # Test regular logging\r
 732     #\r
 733     process = Process(description="a process", cmd=cmd, args=args)\r
 734 \r
 735     #\r
 736     # Test report generation and writing a log\r
 737     #\r
 738     processList = ProcessList("a process list")\r
 739     processList.processes.append(process)\r
 740     processList.echo = True\r
 741     processList.execute()\r
 742 \r
 743     processList.writeLogToDisk(logFilename)\r
 744 \r
 745 # main\r
 746 \r
 747 if __name__ == '__main__':\r
 748     main()\r