aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status\r
   6 of a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['readText',\r
  21            'writeText',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def readText(textFile):\r
  28     if (textFile != ""):\r
  29         fp = open(textFile, 'rb')\r
  30         # Create a text/plain message\r
  31         text = (fp.read())\r
  32         fp.close()\r
  33     return text\r
  34 \r
  35 \r
  36 # readText\r
  37 \r
  38 def writeText(text, textFile):\r
  39     if (textFile != ""):\r
  40         fp = open(textFile, 'wb')\r
  41         # Create a text/plain message\r
  42         fp.write(text)\r
  43         fp.close()\r
  44     return text\r
  45 \r
  46 \r
  47 # readText\r
  48 \r
  49 class Process:\r
  50     """\r
  51     A process with logged output\r
  52     """\r
  53 \r
  54     def __init__(self,\r
  55                  description=None,\r
  56                  cmd=None,\r
  57                  args=[],\r
  58                  cwd=None,\r
  59                  env=None,\r
  60                  batchWrapper=False):\r
  61         """Initialize the standard class variables"""\r
  62         self.cmd = cmd\r
  63         if not description:\r
  64             self.description = cmd\r
  65         else:\r
  66             self.description = description\r
  67         self.status = None\r
  68         self.args = args\r
  69         self.start = None\r
  70         self.end = None\r
  71         self.log = []\r
  72         self.echo = True\r
  73         self.cwd = cwd\r
  74         self.env = env\r
  75         self.batchWrapper = batchWrapper\r
  76         self.processKeys = []\r
  77 \r
  78     # __init__\r
  79 \r
  80     def getElapsedSeconds(self):\r
  81         import math\r
  82 \r
  83         if self.end and self.start:\r
  84             delta = (self.end - self.start)\r
  85             formatted = "%s.%s" % (delta.days * 86400 + delta.seconds,\r
  86                                    int(math.floor(delta.microseconds / 1e3)))\r
  87         else:\r
  88             formatted = None\r
  89         return formatted\r
  90 \r
  91     # getElapsedtime\r
  92 \r
  93     def writeKey(self, writeDict, key=None, value=None, startStop=None):\r
  94         "Write a key, value pair in a supported format"\r
  95         if key != None and (value != None or startStop != None):\r
  96             indent = '\t' * writeDict['indentationLevel']\r
  97             if writeDict['format'] == 'xml':\r
  98                 if startStop == 'start':\r
  99                     writeDict['logHandle'].write("%s<%s>\n" % (indent, key))\r
 100                 elif startStop == 'stop':\r
 101                     writeDict['logHandle'].write("%s</%s>\n" % (indent, key))\r
 102                 else:\r
 103                     writeDict['logHandle'].write(\r
 104                         "%s<%s>%s</%s>\n" % (indent, key, value, key))\r
 105             else:  # writeDict['format'] == 'txt':\r
 106                 writeDict['logHandle'].write(\r
 107                     "%s%40s : %s\n" % (indent, key, value))\r
 108 \r
 109     def writeLogHeader(self, writeDict):\r
 110         import platform\r
 111 \r
 112         # Retrieve operating environment information\r
 113         user = None\r
 114         try:\r
 115             user = os.getlogin()\r
 116         except:\r
 117             try:\r
 118                 user = os.getenv("USERNAME")\r
 119                 if user == None:\r
 120                     user = os.getenv("USER")\r
 121             except:\r
 122                 user = "unknown_user"\r
 123         try:\r
 124             (sysname, nodename, release, version, machine,\r
 125              processor) = platform.uname()\r
 126         except:\r
 127             (sysname, nodename, release, version, machine, processor) = (\r
 128                 "unknown_sysname", "unknown_nodename", "unknown_release",\r
 129                 "unknown_version", "unknown_machine", "unknown_processor")\r
 130         try:\r
 131             hostname = platform.node()\r
 132         except:\r
 133             hostname = "unknown_hostname"\r
 134 \r
 135         self.writeKey(writeDict, 'process', None, 'start')\r
 136         writeDict['indentationLevel'] += 1\r
 137 \r
 138         self.writeKey(writeDict, 'description', self.description)\r
 139         self.writeKey(writeDict, 'cmd', self.cmd)\r
 140         if self.args: self.writeKey(writeDict, 'args', ' '.join(self.args))\r
 141         self.writeKey(writeDict, 'start', self.start)\r
 142         self.writeKey(writeDict, 'end', self.end)\r
 143         self.writeKey(writeDict, 'elapsed', self.getElapsedSeconds())\r
 144 \r
 145         self.writeKey(writeDict, 'user', user)\r
 146         self.writeKey(writeDict, 'sysname', sysname)\r
 147         self.writeKey(writeDict, 'nodename', nodename)\r
 148         self.writeKey(writeDict, 'release', release)\r
 149         self.writeKey(writeDict, 'version', version)\r
 150         self.writeKey(writeDict, 'machine', machine)\r
 151         self.writeKey(writeDict, 'processor', processor)\r
 152 \r
 153         if len(self.processKeys) > 0:\r
 154             self.writeKey(writeDict, 'processKeys', None, 'start')\r
 155             for pair in self.processKeys:\r
 156                 (key, value) = pair\r
 157                 writeDict['indentationLevel'] += 1\r
 158                 self.writeKey(writeDict, key, value)\r
 159                 writeDict['indentationLevel'] -= 1\r
 160             self.writeKey(writeDict, 'processKeys', None, 'stop')\r
 161 \r
 162         self.writeKey(writeDict, 'status', self.status)\r
 163 \r
 164     # writeLogHeader\r
 165 \r
 166     def writeLogFooter(self, writeDict):\r
 167         writeDict['indentationLevel'] -= 1\r
 168         self.writeKey(writeDict, 'process', None, 'stop')\r
 169 \r
 170     # writeLogFooter\r
 171 \r
 172     def writeLog(self, logHandle=sys.stdout, indentationLevel=0, format='xml'):\r
 173         """\r
 174         Write logging information to the specified handle\r
 175         """\r
 176 \r
 177         writeDict = {}\r
 178         writeDict['logHandle'] = logHandle\r
 179         writeDict['indentationLevel'] = indentationLevel\r
 180         writeDict['format'] = format\r
 181 \r
 182         if logHandle:\r
 183             self.writeLogHeader(writeDict)\r
 184 \r
 185             if self.log:\r
 186                 self.writeKey(writeDict, 'output', None, 'start')\r
 187                 if format == 'xml':\r
 188                     logHandle.write("<![CDATA[\n")\r
 189                 for line in self.log:\r
 190                     logHandle.write('%s%s\n' % ("", line))\r
 191                 if format == 'xml':\r
 192                     logHandle.write("]]>\n")\r
 193                 self.writeKey(writeDict, 'output', None, 'stop')\r
 194 \r
 195             self.writeLogFooter(writeDict)\r
 196 \r
 197     # writeLog\r
 198 \r
 199     def writeLogToDisk(self, logFilename=None, format='xml', header=None):\r
 200         if logFilename:\r
 201             try:\r
 202                 # This also doesn't seem like the best structure...\r
 203                 # 3.1\r
 204                 try:\r
 205                     logHandle = open(logFilename, mode='wt', encoding="utf-8")\r
 206                 # 2.6\r
 207                 except:\r
 208                     logHandle = open(logFilename, mode='wt')\r
 209             except:\r
 210                 print("Couldn't open log : %s" % logFilename)\r
 211                 logHandle = None\r
 212 \r
 213         if logHandle:\r
 214             if header:\r
 215                 if format == 'xml':\r
 216                     logHandle.write("<![CDATA[\n")\r
 217                 logHandle.write(header)\r
 218                 if format == 'xml':\r
 219                     logHandle.write("]]>\n")\r
 220             self.writeLog(logHandle)\r
 221             logHandle.close()\r
 222 \r
 223     # writeLogToDisk\r
 224 \r
 225     def logLine(self, line):\r
 226         "Add a line of text to the log"\r
 227         self.log.append(line.rstrip())\r
 228         if self.echo:\r
 229             print("%s" % line.rstrip())\r
 230 \r
 231     # logLine\r
 232 \r
 233     def execute(self):\r
 234         """\r
 235         Execute this process\r
 236         """\r
 237         import re\r
 238         import datetime\r
 239         import traceback\r
 240 \r
 241         try:\r
 242             import subprocess as sp\r
 243         except:\r
 244             sp = None\r
 245 \r
 246         self.start = datetime.datetime.now()\r
 247 \r
 248         cmdargs = [self.cmd]\r
 249         cmdargs.extend(self.args)\r
 250 \r
 251         if self.echo:\r
 252             if sp:\r
 253                 print(\r
 254                     "\n%s : %s\n" % (self.__class__, sp.list2cmdline(cmdargs)))\r
 255             else:\r
 256                 print("\n%s : %s\n" % (self.__class__, " ".join(cmdargs)))\r
 257 \r
 258         # intialize a few variables that may or may not be set later\r
 259         process = None\r
 260         tmpWrapper = None\r
 261         stdout = None\r
 262         stdin = None\r
 263         parentenv = os.environ\r
 264         parentcwd = os.getcwd()\r
 265 \r
 266         try:\r
 267             # Using subprocess\r
 268             if sp:\r
 269                 if self.batchWrapper:\r
 270                     cmd = " ".join(cmdargs)\r
 271                     tmpWrapper = os.path.join(self.cwd, "process.bat")\r
 272                     writeText(cmd, tmpWrapper)\r
 273                     print("%s : Running process through wrapper %s\n" % (\r
 274                         self.__class__, tmpWrapper))\r
 275                     process = sp.Popen([tmpWrapper], stdout=sp.PIPE,\r
 276                                        stderr=sp.STDOUT,\r
 277                                        cwd=self.cwd, env=self.env)\r
 278                 else:\r
 279                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 280                                        stderr=sp.STDOUT,\r
 281                                        cwd=self.cwd, env=self.env)\r
 282 \r
 283             # using os.popen4\r
 284             else:\r
 285                 if self.env:\r
 286                     os.environ = self.env\r
 287                 if self.cwd:\r
 288                     os.chdir(self.cwd)\r
 289 \r
 290                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 291         except:\r
 292             print("Couldn't execute command : %s" % cmdargs[0])\r
 293             traceback.print_exc()\r
 294 \r
 295         # Using subprocess\r
 296         if sp:\r
 297             if process != None:\r
 298                 # pid = process.pid\r
 299                 # log.logLine("process id %s\n" % pid)\r
 300 \r
 301                 try:\r
 302                     # This is more proper python, and resolves some issues with\r
 303                     # a process ending before all of its output has been\r
 304                     # processed, but it also seems to stall when the read\r
 305                     # buffer is near or over it's limit. this happens\r
 306                     # relatively frequently with processes that generate lots\r
 307                     # of print statements.\r
 308                     for line in process.stdout:\r
 309                         self.logLine(line)\r
 310                     #\r
 311                     # So we go with the, um, uglier  option below\r
 312 \r
 313                     # This is now used to ensure that the process has finished\r
 314                     line = ""\r
 315                     while line != None and process.poll() == None:\r
 316                         try:\r
 317                             line = process.stdout.readline()\r
 318                         except:\r
 319                             break\r
 320                         # 3.1\r
 321                         try:\r
 322                             self.logLine(str(line, encoding="utf-8"))\r
 323                         # 2.6\r
 324                         except:\r
 325                             self.logLine(line)\r
 326                 except:\r
 327                     self.logLine("Logging error : %s" % sys.exc_info()[0])\r
 328 \r
 329                 self.status = process.returncode\r
 330 \r
 331                 if self.batchWrapper and tmpWrapper:\r
 332                     try:\r
 333                         os.remove(tmpWrapper)\r
 334                     except:\r
 335                         print("Couldn't remove temp wrapper : %s" % tmpWrapper)\r
 336                         traceback.print_exc()\r
 337 \r
 338         # Using os.popen4\r
 339         else:\r
 340             exitCode = -1\r
 341             try:\r
 342                 # print("reading stdout lines")\r
 343                 stdoutLines = stdout.readlines()\r
 344                 exitCode = stdout.close()\r
 345 \r
 346                 stdout.close()\r
 347                 stdin.close()\r
 348 \r
 349                 if self.env:\r
 350                     os.environ = parentenv\r
 351                 if self.cwd:\r
 352                     os.chdir(parentcwd)\r
 353 \r
 354                 if len(stdoutLines) > 0:\r
 355                     for line in stdoutLines:\r
 356                         self.logLine(line)\r
 357 \r
 358                 if not exitCode:\r
 359                     exitCode = 0\r
 360             except:\r
 361                 self.logLine("Logging error : %s" % sys.exc_info()[0])\r
 362 \r
 363             self.status = exitCode\r
 364 \r
 365         self.end = datetime.datetime.now()\r
 366         # execute\r
 367 \r
 368 \r
 369 # Process\r
 370 \r
 371 class ProcessList(Process):\r
 372     """\r
 373     A list of processes with logged output\r
 374     """\r
 375 \r
 376     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 377         Process.__init__(self, description, None, None, cwd, env)\r
 378         "Initialize the standard class variables"\r
 379         self.processes = []\r
 380         self.blocking = blocking\r
 381 \r
 382     # __init__\r
 383 \r
 384     def generateReport(self, writeDict):\r
 385         """\r
 386         Generate a log based on the success of the child processes\r
 387         """\r
 388         if self.processes:\r
 389             _status = True\r
 390             indent = '\t' * (writeDict['indentationLevel'] + 1)\r
 391 \r
 392             self.log = []\r
 393 \r
 394             for child in self.processes:\r
 395                 if isinstance(child, ProcessList):\r
 396                     child.generateReport(writeDict)\r
 397 \r
 398                 childResult = ""\r
 399                 key = child.description\r
 400                 value = child.status\r
 401                 if writeDict['format'] == 'xml':\r
 402                     childResult = (\r
 403                         "%s<result description=\"%s\">%s</result>" % (\r
 404                             indent, key, value))\r
 405                 else:  # writeDict['format'] == 'txt':\r
 406                     childResult = ("%s%40s : %s" % (indent, key, value))\r
 407                 self.log.append(childResult)\r
 408 \r
 409                 if child.status != 0:\r
 410                     _status = False\r
 411             if not _status:\r
 412                 self.status = -1\r
 413             else:\r
 414                 self.status = 0\r
 415         else:\r
 416             self.log = ["No child processes available to generate a report"]\r
 417             self.status = -1\r
 418 \r
 419     def writeLogHeader(self, writeDict):\r
 420         self.writeKey(writeDict, 'processList', None, 'start')\r
 421         writeDict['indentationLevel'] += 1\r
 422 \r
 423         self.writeKey(writeDict, 'description', self.description)\r
 424         self.writeKey(writeDict, 'start', self.start)\r
 425         self.writeKey(writeDict, 'end', self.end)\r
 426         self.writeKey(writeDict, 'elapsed', self.getElapsedSeconds())\r
 427 \r
 428         self.generateReport(writeDict)\r
 429 \r
 430         self.writeKey(writeDict, 'status', self.status)\r
 431 \r
 432     # writeLogHeader\r
 433 \r
 434     def writeLogFooter(self, writeDict):\r
 435         writeDict['indentationLevel'] -= 1\r
 436         self.writeKey(writeDict, 'processList', None, 'stop')\r
 437 \r
 438     # writeLogFooter\r
 439 \r
 440     def writeLog(self, logHandle=sys.stdout, indentationLevel=0, format='xml'):\r
 441         """\r
 442         Write logging information to the specified handle\r
 443         """\r
 444 \r
 445         writeDict = {}\r
 446         writeDict['logHandle'] = logHandle\r
 447         writeDict['indentationLevel'] = indentationLevel\r
 448         writeDict['format'] = format\r
 449 \r
 450         if logHandle:\r
 451             self.writeLogHeader(writeDict)\r
 452 \r
 453             if self.log:\r
 454                 self.writeKey(writeDict, 'output', None, 'start')\r
 455                 for line in self.log:\r
 456                     logHandle.write('%s%s\n' % ("", line))\r
 457                 self.writeKey(writeDict, 'output', None, 'stop')\r
 458 \r
 459             if self.processes:\r
 460                 self.writeKey(writeDict, 'processes', None, 'start')\r
 461                 for child in self.processes:\r
 462                     child.writeLog(logHandle, indentationLevel + 1, format)\r
 463                 self.writeKey(writeDict, 'processes', None, 'stop')\r
 464 \r
 465             self.writeLogFooter(writeDict)\r
 466 \r
 467     # writeLog\r
 468 \r
 469     def execute(self):\r
 470         """\r
 471         Execute this list of processes\r
 472         """\r
 473         import datetime\r
 474 \r
 475         self.start = datetime.datetime.now()\r
 476 \r
 477         self.status = 0\r
 478         if self.processes:\r
 479             for child in self.processes:\r
 480                 if child:\r
 481                     try:\r
 482                         child.execute()\r
 483                     except:\r
 484                         print("%s : caught exception in child class %s" % (\r
 485                             self.__class__, child.__class__))\r
 486                         traceback.print_exc()\r
 487                         child.status = -1\r
 488 \r
 489                     if self.blocking and child.status != 0:\r
 490                         print("%s : child class %s finished with an error" % (\r
 491                             self.__class__, child.__class__))\r
 492                         self.status = -1\r
 493                         break\r
 494 \r
 495         self.end = datetime.datetime.now()\r
 496         # execute\r
 497 \r
 498 \r
 499 # ProcessList\r
 500 \r
 501 def main():\r
 502     import optparse\r
 503 \r
 504     p = optparse.OptionParser(description='A process logging script',\r
 505                               prog='process',\r
 506                               version='process 0.1',\r
 507                               usage=('%prog [options] '\r
 508                                      '[options for the logged process]'))\r
 509     p.add_option('--cmd', '-c', default=None)\r
 510     p.add_option('--log', '-l', default=None)\r
 511 \r
 512     options, arguments = p.parse_args()\r
 513 \r
 514     #\r
 515     # Get options\r
 516     # \r
 517     cmd = options.cmd\r
 518     logFilename = options.log\r
 519 \r
 520     try:\r
 521         argsStart = sys.argv.index('--') + 1\r
 522         args = sys.argv[argsStart:]\r
 523     except:\r
 524         argsStart = len(sys.argv) + 1\r
 525         args = []\r
 526 \r
 527     if cmd == None:\r
 528         print("process: No command specified")\r
 529 \r
 530     #\r
 531     # Test regular logging\r
 532     #\r
 533     process = Process(description="a process", cmd=cmd, args=args)\r
 534 \r
 535     #\r
 536     # Test report generation and writing a log\r
 537     #\r
 538     processList = ProcessList("a process list")\r
 539     processList.processes.append(process)\r
 540     processList.echo = True\r
 541     processList.execute()\r
 542 \r
 543     processList.writeLogToDisk(logFilename)\r
 544 \r
 545 # main\r
 546 \r
 547 if __name__ == '__main__':\r
 548     main()\r