aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 from __future__ import division\r
  10 \r
  11 import os\r
  12 import sys\r
  13 import traceback\r
  14 \r
  15 __author__ = 'ACES Developers'\r
  16 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  17 __license__ = ''\r
  18 __maintainer__ = 'ACES Developers'\r
  19 __email__ = 'aces@oscars.org'\r
  20 __status__ = 'Production'\r
  21 \r
  22 __all__ = ['read_text',\r
  23            'write_text',\r
  24            'Process',\r
  25            'ProcessList',\r
  26            'main']\r
  27 \r
  28 \r
  29 def read_text(text_file):\r
  30     """\r
  31     Object description.\r
  32 \r
  33     Parameters\r
  34     ----------\r
  35     parameter : type\r
  36         Parameter description.\r
  37 \r
  38     Returns\r
  39     -------\r
  40     type\r
  41          Return value description.\r
  42     """\r
  43 \r
  44     if text_file != '':\r
  45         with open(text_file, 'rb') as fp:\r
  46             text = (fp.read())\r
  47     return text\r
  48 \r
  49 \r
  50 def write_text(text, text_file):\r
  51     """\r
  52     Object description.\r
  53 \r
  54     Parameters\r
  55     ----------\r
  56     parameter : type\r
  57         Parameter description.\r
  58 \r
  59     Returns\r
  60     -------\r
  61     type\r
  62          Return value description.\r
  63     """\r
  64 \r
  65     if text_file != '':\r
  66         with open(text_file, 'wb') as fp:\r
  67             fp.write(text)\r
  68     return text\r
  69 \r
  70 \r
  71 class Process:\r
  72     """\r
  73     A process with logged output.\r
  74     """\r
  75 \r
  76     def __init__(self,\r
  77                  description=None,\r
  78                  cmd=None,\r
  79                  args=None,\r
  80                  cwd=None,\r
  81                  env=None,\r
  82                  batch_wrapper=False):\r
  83         """\r
  84         Initialize the standard class variables.\r
  85 \r
  86         Parameters\r
  87         ----------\r
  88         parameter : type\r
  89             Parameter description.\r
  90 \r
  91         Returns\r
  92         -------\r
  93         type\r
  94              Return value description.\r
  95         """\r
  96 \r
  97         if args is None:\r
  98             args = []\r
  99 \r
 100         self.cmd = cmd\r
 101         if not description:\r
 102             self.description = cmd\r
 103         else:\r
 104             self.description = description\r
 105         self.status = None\r
 106         self.args = args\r
 107         self.start = None\r
 108         self.end = None\r
 109         self.log = []\r
 110         self.echo = True\r
 111         self.cwd = cwd\r
 112         self.env = env\r
 113         self.batch_wrapper = batch_wrapper\r
 114         self.process_keys = []\r
 115 \r
 116     def get_elapsed_seconds(self):\r
 117         """\r
 118         Object description.\r
 119 \r
 120         Parameters\r
 121         ----------\r
 122         parameter : type\r
 123             Parameter description.\r
 124 \r
 125         Returns\r
 126         -------\r
 127         type\r
 128              Return value description.\r
 129         """\r
 130 \r
 131         import math\r
 132 \r
 133         if self.end and self.start:\r
 134             delta = (self.end - self.start)\r
 135             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 136                                    int(math.floor(delta.microseconds / 1e3)))\r
 137         else:\r
 138             formatted = None\r
 139         return formatted\r
 140 \r
 141     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 142         """\r
 143         Writes a key / value pair in a supported format.\r
 144 \r
 145         Parameters\r
 146         ----------\r
 147         parameter : type\r
 148             Parameter description.\r
 149 \r
 150         Returns\r
 151         -------\r
 152         type\r
 153              Return value description.\r
 154         """\r
 155 \r
 156         if key is not None and (value is not None or start_stop is not None):\r
 157             indent = '\t' * write_dict['indentationLevel']\r
 158             if write_dict['format'] == 'xml':\r
 159                 if start_stop == 'start':\r
 160                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 161                 elif start_stop == 'stop':\r
 162                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 163                 else:\r
 164                     write_dict['logHandle'].write(\r
 165                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 166             else:\r
 167                 write_dict['logHandle'].write(\r
 168                     '%s%40s : %s\n' % (indent, key, value))\r
 169 \r
 170     def write_log_header(self, write_dict):\r
 171         """\r
 172         Object description.\r
 173 \r
 174         Parameters\r
 175         ----------\r
 176         parameter : type\r
 177             Parameter description.\r
 178 \r
 179         Returns\r
 180         -------\r
 181         type\r
 182              Return value description.\r
 183         """\r
 184 \r
 185         import platform\r
 186 \r
 187         try:\r
 188             user = os.getlogin()\r
 189         except:\r
 190             try:\r
 191                 user = os.getenv('USERNAME')\r
 192                 if user is None:\r
 193                     user = os.getenv('USER')\r
 194             except:\r
 195                 user = 'unknown_user'\r
 196         try:\r
 197             (sysname, nodename, release, version, machine,\r
 198              processor) = platform.uname()\r
 199         except:\r
 200             (sysname, nodename, release, version, machine, processor) = (\r
 201                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 202                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 203 \r
 204         self.write_key(write_dict, 'process', None, 'start')\r
 205         write_dict['indentationLevel'] += 1\r
 206 \r
 207         self.write_key(write_dict, 'description', self.description)\r
 208         self.write_key(write_dict, 'cmd', self.cmd)\r
 209         if self.args:\r
 210             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 211         self.write_key(write_dict, 'start', self.start)\r
 212         self.write_key(write_dict, 'end', self.end)\r
 213         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 214 \r
 215         self.write_key(write_dict, 'user', user)\r
 216         self.write_key(write_dict, 'sysname', sysname)\r
 217         self.write_key(write_dict, 'nodename', nodename)\r
 218         self.write_key(write_dict, 'release', release)\r
 219         self.write_key(write_dict, 'version', version)\r
 220         self.write_key(write_dict, 'machine', machine)\r
 221         self.write_key(write_dict, 'processor', processor)\r
 222 \r
 223         if len(self.process_keys) > 0:\r
 224             self.write_key(write_dict, 'processKeys', None, 'start')\r
 225             for pair in self.process_keys:\r
 226                 (key, value) = pair\r
 227                 write_dict['indentationLevel'] += 1\r
 228                 self.write_key(write_dict, key, value)\r
 229                 write_dict['indentationLevel'] -= 1\r
 230             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 231 \r
 232         self.write_key(write_dict, 'status', self.status)\r
 233 \r
 234     def write_log_footer(self, write_dict):\r
 235         """\r
 236         Object description.\r
 237 \r
 238         Parameters\r
 239         ----------\r
 240         parameter : type\r
 241             Parameter description.\r
 242 \r
 243         Returns\r
 244         -------\r
 245         type\r
 246              Return value description.\r
 247         """\r
 248 \r
 249         write_dict['indentationLevel'] -= 1\r
 250         self.write_key(write_dict, 'process', None, 'stop')\r
 251 \r
 252     def write_log(self,\r
 253                   log_handle=sys.stdout,\r
 254                   indentation_level=0,\r
 255                   format='xml'):\r
 256         """\r
 257         Writes logging information to the specified handle.\r
 258 \r
 259         Parameters\r
 260         ----------\r
 261         parameter : type\r
 262             Parameter description.\r
 263 \r
 264         Returns\r
 265         -------\r
 266         type\r
 267              Return value description.\r
 268         """\r
 269 \r
 270         write_dict = {}\r
 271         write_dict['logHandle'] = log_handle\r
 272         write_dict['indentationLevel'] = indentation_level\r
 273         write_dict['format'] = format\r
 274 \r
 275         if log_handle:\r
 276             self.write_log_header(write_dict)\r
 277 \r
 278             if self.log:\r
 279                 self.write_key(write_dict, 'output', None, 'start')\r
 280                 if format == 'xml':\r
 281                     log_handle.write('<![CDATA[\n')\r
 282                 for line in self.log:\r
 283                     log_handle.write('%s%s\n' % ('', line))\r
 284                 if format == 'xml':\r
 285                     log_handle.write(']]>\n')\r
 286                 self.write_key(write_dict, 'output', None, 'stop')\r
 287 \r
 288             self.write_log_footer(write_dict)\r
 289 \r
 290     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 291         """\r
 292         Object description.\r
 293 \r
 294         Parameters\r
 295         ----------\r
 296         parameter : type\r
 297             Parameter description.\r
 298 \r
 299         Returns\r
 300         -------\r
 301         type\r
 302              Return value description.\r
 303         """\r
 304 \r
 305         if log_filename:\r
 306             try:\r
 307                 # TODO: Review statements.\r
 308                 # 3.1\r
 309                 try:\r
 310                     log_handle = (\r
 311                         open(log_filename, mode='wt', encoding='utf-8'))\r
 312                 # 2.6\r
 313                 except:\r
 314                     log_handle = open(log_filename, mode='wt')\r
 315             except:\r
 316                 print('Couldn\'t open log : %s' % log_filename)\r
 317                 log_handle = None\r
 318 \r
 319         if log_handle:\r
 320             if header:\r
 321                 if format == 'xml':\r
 322                     log_handle.write('<![CDATA[\n')\r
 323                 log_handle.write(header)\r
 324                 if format == 'xml':\r
 325                     log_handle.write(']]>\n')\r
 326             self.write_log(log_handle)\r
 327             log_handle.close()\r
 328 \r
 329     def log_line(self, line):\r
 330         """\r
 331         Adds a line of text to the log.\r
 332 \r
 333         Parameters\r
 334         ----------\r
 335         parameter : type\r
 336             Parameter description.\r
 337 \r
 338         Returns\r
 339         -------\r
 340         type\r
 341              Return value description.\r
 342         """\r
 343 \r
 344         self.log.append(line.rstrip())\r
 345         if self.echo:\r
 346             print('%s' % line.rstrip())\r
 347 \r
 348     def execute(self):\r
 349         """\r
 350         Executes the current process.\r
 351 \r
 352         Parameters\r
 353         ----------\r
 354         parameter : type\r
 355             Parameter description.\r
 356 \r
 357         Returns\r
 358         -------\r
 359         type\r
 360              Return value description.\r
 361         """\r
 362 \r
 363         import datetime\r
 364         import traceback\r
 365 \r
 366         try:\r
 367             import subprocess as sp\r
 368         except:\r
 369             sp = None\r
 370 \r
 371         self.start = datetime.datetime.now()\r
 372 \r
 373         cmdargs = [self.cmd]\r
 374         cmdargs.extend(self.args)\r
 375 \r
 376         if self.echo:\r
 377             if sp:\r
 378                 print(\r
 379                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 380             else:\r
 381                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 382 \r
 383         process = None\r
 384         tmp_wrapper = None\r
 385         stdout = None\r
 386         stdin = None\r
 387         parentenv = os.environ\r
 388         parentcwd = os.getcwd()\r
 389 \r
 390         try:\r
 391             # Using *subprocess*.\r
 392             if sp:\r
 393                 if self.batch_wrapper:\r
 394                     cmd = ' '.join(cmdargs)\r
 395                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 396                     write_text(cmd, tmp_wrapper)\r
 397                     print('%s : Running process through wrapper %s\n' % (\r
 398                         self.__class__, tmp_wrapper))\r
 399                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 400                                        stderr=sp.STDOUT,\r
 401                                        cwd=self.cwd, env=self.env)\r
 402                 else:\r
 403                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 404                                        stderr=sp.STDOUT,\r
 405                                        cwd=self.cwd, env=self.env)\r
 406 \r
 407             # using *os.popen4*.\r
 408             else:\r
 409                 if self.env:\r
 410                     os.environ = self.env\r
 411                 if self.cwd:\r
 412                     os.chdir(self.cwd)\r
 413 \r
 414                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 415         except:\r
 416             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 417             traceback.print_exc()\r
 418 \r
 419         # Using *subprocess*\r
 420         if sp:\r
 421             if process is not None:\r
 422                 # pid = process.pid\r
 423                 # log.logLine('process id %s\n' % pid)\r
 424 \r
 425                 try:\r
 426                     # This is more proper python, and resolves some issues with\r
 427                     # a process ending before all of its output has been\r
 428                     # processed, but it also seems to stall when the read\r
 429                     # buffer is near or over its limit. This happens\r
 430                     # relatively frequently with processes that generate lots\r
 431                     # of print statements.\r
 432                     for line in process.stdout:\r
 433                         self.log_line(line)\r
 434 \r
 435                     # So we go with the, um, uglier option below.\r
 436 \r
 437                     # This is now used to ensure that the process has finished.\r
 438                     line = ''\r
 439                     while line is not None and process.poll() is None:\r
 440                         try:\r
 441                             line = process.stdout.readline()\r
 442                         except:\r
 443                             break\r
 444                         # 3.1\r
 445                         try:\r
 446                             # TODO: Investigate previous eroneous statement.\r
 447                             # self.log_line(str(line, encoding='utf-8'))\r
 448                             self.log_line(str(line))\r
 449                         # 2.6\r
 450                         except:\r
 451                             self.log_line(line)\r
 452                 except:\r
 453                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 454 \r
 455                 self.status = process.returncode\r
 456 \r
 457                 if self.batch_wrapper and tmp_wrapper:\r
 458                     try:\r
 459                         os.remove(tmp_wrapper)\r
 460                     except:\r
 461                         print(\r
 462                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 463                         traceback.print_exc()\r
 464 \r
 465         # Using *os.popen4*.\r
 466         else:\r
 467             exit_code = -1\r
 468             try:\r
 469                 stdout_lines = stdout.readlines()\r
 470                 # TODO: Investigate if this is the good behavior, close() does\r
 471                 # not return anything / None.\r
 472                 exit_code = stdout.close()\r
 473 \r
 474                 stdout.close()\r
 475                 stdin.close()\r
 476 \r
 477                 if self.env:\r
 478                     os.environ = parentenv\r
 479                 if self.cwd:\r
 480                     os.chdir(parentcwd)\r
 481 \r
 482                 if len(stdout_lines) > 0:\r
 483                     for line in stdout_lines:\r
 484                         self.log_line(line)\r
 485 \r
 486                 if not exit_code:\r
 487                     exit_code = 0\r
 488             except:\r
 489                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 490 \r
 491             self.status = exit_code\r
 492 \r
 493         self.end = datetime.datetime.now()\r
 494 \r
 495 \r
 496 class ProcessList(Process):\r
 497     """\r
 498     A list of processes with logged output.\r
 499     """\r
 500 \r
 501     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 502         """\r
 503         Object description.\r
 504 \r
 505         Parameters\r
 506         ----------\r
 507         parameter : type\r
 508             Parameter description.\r
 509 \r
 510         Returns\r
 511         -------\r
 512         type\r
 513              Return value description.\r
 514         """\r
 515 \r
 516         Process.__init__(self, description, None, None, cwd, env)\r
 517         'Initialize the standard class variables'\r
 518         self.processes = []\r
 519         self.blocking = blocking\r
 520 \r
 521     def generate_report(self, write_dict):\r
 522         """\r
 523         Generates a log based on the success of the child processes.\r
 524 \r
 525         Parameters\r
 526         ----------\r
 527         parameter : type\r
 528             Parameter description.\r
 529 \r
 530         Returns\r
 531         -------\r
 532         type\r
 533              Return value description.\r
 534         """\r
 535 \r
 536         if self.processes:\r
 537             _status = True\r
 538             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 539 \r
 540             self.log = []\r
 541 \r
 542             for child in self.processes:\r
 543                 if isinstance(child, ProcessList):\r
 544                     child.generate_report(write_dict)\r
 545 \r
 546                 key = child.description\r
 547                 value = child.status\r
 548                 if write_dict['format'] == 'xml':\r
 549                     child_result = (\r
 550                         '%s<result description=\'%s\'>%s</result>' % (\r
 551                             indent, key, value))\r
 552                 else:\r
 553                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 554                 self.log.append(child_result)\r
 555 \r
 556                 if child.status != 0:\r
 557                     _status = False\r
 558             if not _status:\r
 559                 self.status = -1\r
 560             else:\r
 561                 self.status = 0\r
 562         else:\r
 563             self.log = ['No child processes available to generate a report']\r
 564             self.status = -1\r
 565 \r
 566     def write_log_header(self, write_dict):\r
 567         """\r
 568         Object description.\r
 569 \r
 570         Parameters\r
 571         ----------\r
 572         parameter : type\r
 573             Parameter description.\r
 574 \r
 575         Returns\r
 576         -------\r
 577         type\r
 578              Return value description.\r
 579         """\r
 580 \r
 581         self.write_key(write_dict, 'processList', None, 'start')\r
 582         write_dict['indentationLevel'] += 1\r
 583 \r
 584         self.write_key(write_dict, 'description', self.description)\r
 585         self.write_key(write_dict, 'start', self.start)\r
 586         self.write_key(write_dict, 'end', self.end)\r
 587         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 588 \r
 589         self.generate_report(write_dict)\r
 590 \r
 591         self.write_key(write_dict, 'status', self.status)\r
 592 \r
 593     def write_log_footer(self, write_dict):\r
 594         """\r
 595         Object description.\r
 596 \r
 597         Parameters\r
 598         ----------\r
 599         parameter : type\r
 600             Parameter description.\r
 601 \r
 602         Returns\r
 603         -------\r
 604         type\r
 605              Return value description.\r
 606         """\r
 607 \r
 608         write_dict['indentationLevel'] -= 1\r
 609         self.write_key(write_dict, 'processList', None, 'stop')\r
 610 \r
 611     def write_log(self,\r
 612                   log_handle=sys.stdout,\r
 613                   indentation_level=0,\r
 614                   format='xml'):\r
 615         """\r
 616         Writes logging information to the specified handle.\r
 617 \r
 618         Parameters\r
 619         ----------\r
 620         parameter : type\r
 621             Parameter description.\r
 622 \r
 623         Returns\r
 624         -------\r
 625         type\r
 626              Return value description.\r
 627         """\r
 628 \r
 629         write_dict = {}\r
 630         write_dict['logHandle'] = log_handle\r
 631         write_dict['indentationLevel'] = indentation_level\r
 632         write_dict['format'] = format\r
 633 \r
 634         if log_handle:\r
 635             self.write_log_header(write_dict)\r
 636 \r
 637             if self.log:\r
 638                 self.write_key(write_dict, 'output', None, 'start')\r
 639                 for line in self.log:\r
 640                     log_handle.write('%s%s\n' % ('', line))\r
 641                 self.write_key(write_dict, 'output', None, 'stop')\r
 642 \r
 643             if self.processes:\r
 644                 self.write_key(write_dict, 'processes', None, 'start')\r
 645                 for child in self.processes:\r
 646                     child.write_log(log_handle, indentation_level + 1, format)\r
 647                 self.write_key(write_dict, 'processes', None, 'stop')\r
 648 \r
 649             self.write_log_footer(write_dict)\r
 650 \r
 651     def execute(self):\r
 652         """\r
 653         Executes the list of processes.\r
 654 \r
 655         Parameters\r
 656         ----------\r
 657         parameter : type\r
 658             Parameter description.\r
 659 \r
 660         Returns\r
 661         -------\r
 662         type\r
 663              Return value description.\r
 664         """\r
 665 \r
 666         import datetime\r
 667 \r
 668         self.start = datetime.datetime.now()\r
 669 \r
 670         self.status = 0\r
 671         if self.processes:\r
 672             for child in self.processes:\r
 673                 if child:\r
 674                     try:\r
 675                         child.execute()\r
 676                     except:\r
 677                         print('%s : caught exception in child class %s' % (\r
 678                             self.__class__, child.__class__))\r
 679                         traceback.print_exc()\r
 680                         child.status = -1\r
 681 \r
 682                     if self.blocking and child.status != 0:\r
 683                         print('%s : child class %s finished with an error' % (\r
 684                             self.__class__, child.__class__))\r
 685                         self.status = -1\r
 686                         break\r
 687 \r
 688         self.end = datetime.datetime.now()\r
 689 \r
 690 \r
 691 def main():\r
 692     """\r
 693     Object description.\r
 694 \r
 695     Parameters\r
 696     ----------\r
 697     parameter : type\r
 698         Parameter description.\r
 699 \r
 700     Returns\r
 701     -------\r
 702     type\r
 703          Return value description.\r
 704     """\r
 705 \r
 706     import optparse\r
 707 \r
 708     p = optparse.OptionParser(description='A process logging script',\r
 709                               prog='process',\r
 710                               version='process 0.1',\r
 711                               usage=('%prog [options] '\r
 712                                      '[options for the logged process]'))\r
 713     p.add_option('--cmd', '-c', default=None)\r
 714     p.add_option('--log', '-l', default=None)\r
 715 \r
 716     options, arguments = p.parse_args()\r
 717 \r
 718     cmd = options.cmd\r
 719     log_filename = options.log\r
 720 \r
 721     try:\r
 722         args_start = sys.argv.index('--') + 1\r
 723         args = sys.argv[args_start:]\r
 724     except:\r
 725         args = []\r
 726 \r
 727     if cmd is None:\r
 728         print('process: No command specified')\r
 729 \r
 730     # Testing regular logging.\r
 731     process = Process(description='a process', cmd=cmd, args=args)\r
 732 \r
 733     # Testing report generation and writing a log.\r
 734     process_list = ProcessList('a process list')\r
 735     process_list.processes.append(process)\r
 736     process_list.echo = True\r
 737     process_list.execute()\r
 738 \r
 739     process_list.write_log_to_disk(log_filename)\r
 740 \r
 741 \r
 742 if __name__ == '__main__':\r
 743     main()\r