aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 import os\r
  10 import sys\r
  11 import traceback\r
  12 \r
  13 __author__ = 'ACES Developers'\r
  14 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  15 __license__ = ''\r
  16 __maintainer__ = 'ACES Developers'\r
  17 __email__ = 'aces@oscars.org'\r
  18 __status__ = 'Production'\r
  19 \r
  20 __all__ = ['read_text',\r
  21            'write_text',\r
  22            'Process',\r
  23            'ProcessList',\r
  24            'main']\r
  25 \r
  26 \r
  27 def read_text(text_file):\r
  28     """\r
  29     Object description.\r
  30 \r
  31     Parameters\r
  32     ----------\r
  33     parameter : type\r
  34         Parameter description.\r
  35 \r
  36     Returns\r
  37     -------\r
  38     type\r
  39          Return value description.\r
  40     """\r
  41 \r
  42     if text_file != '':\r
  43         with open(text_file, 'rb') as fp:\r
  44             text = (fp.read())\r
  45     return text\r
  46 \r
  47 \r
  48 def write_text(text, text_file):\r
  49     """\r
  50     Object description.\r
  51 \r
  52     Parameters\r
  53     ----------\r
  54     parameter : type\r
  55         Parameter description.\r
  56 \r
  57     Returns\r
  58     -------\r
  59     type\r
  60          Return value description.\r
  61     """\r
  62 \r
  63     if text_file != '':\r
  64         with open(text_file, 'wb') as fp:\r
  65             fp.write(text)\r
  66     return text\r
  67 \r
  68 \r
  69 class Process:\r
  70     """\r
  71     A process with logged output.\r
  72     """\r
  73 \r
  74     def __init__(self,\r
  75                  description=None,\r
  76                  cmd=None,\r
  77                  args=[],\r
  78                  cwd=None,\r
  79                  env=None,\r
  80                  batch_wrapper=False):\r
  81         """\r
  82         Initialize the standard class variables.\r
  83 \r
  84         Parameters\r
  85         ----------\r
  86         parameter : type\r
  87             Parameter description.\r
  88 \r
  89         Returns\r
  90         -------\r
  91         type\r
  92              Return value description.\r
  93         """\r
  94 \r
  95         self.cmd = cmd\r
  96         if not description:\r
  97             self.description = cmd\r
  98         else:\r
  99             self.description = description\r
 100         self.status = None\r
 101         self.args = args\r
 102         self.start = None\r
 103         self.end = None\r
 104         self.log = []\r
 105         self.echo = True\r
 106         self.cwd = cwd\r
 107         self.env = env\r
 108         self.batch_wrapper = batch_wrapper\r
 109         self.process_keys = []\r
 110 \r
 111     def get_elapsed_seconds(self):\r
 112         """\r
 113         Object description.\r
 114 \r
 115         Parameters\r
 116         ----------\r
 117         parameter : type\r
 118             Parameter description.\r
 119 \r
 120         Returns\r
 121         -------\r
 122         type\r
 123              Return value description.\r
 124         """\r
 125 \r
 126         import math\r
 127 \r
 128         if self.end and self.start:\r
 129             delta = (self.end - self.start)\r
 130             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 131                                    int(math.floor(delta.microseconds / 1e3)))\r
 132         else:\r
 133             formatted = None\r
 134         return formatted\r
 135 \r
 136     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 137         """\r
 138         Writes a key / value pair in a supported format.\r
 139 \r
 140         Parameters\r
 141         ----------\r
 142         parameter : type\r
 143             Parameter description.\r
 144 \r
 145         Returns\r
 146         -------\r
 147         type\r
 148              Return value description.\r
 149         """\r
 150 \r
 151         if key != None and (value != None or start_stop != None):\r
 152             indent = '\t' * write_dict['indentationLevel']\r
 153             if write_dict['format'] == 'xml':\r
 154                 if start_stop == 'start':\r
 155                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 156                 elif start_stop == 'stop':\r
 157                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 158                 else:\r
 159                     write_dict['logHandle'].write(\r
 160                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 161             else:\r
 162                 write_dict['logHandle'].write(\r
 163                     '%s%40s : %s\n' % (indent, key, value))\r
 164 \r
 165     def write_log_header(self, write_dict):\r
 166         """\r
 167         Object description.\r
 168 \r
 169         Parameters\r
 170         ----------\r
 171         parameter : type\r
 172             Parameter description.\r
 173 \r
 174         Returns\r
 175         -------\r
 176         type\r
 177              Return value description.\r
 178         """\r
 179 \r
 180         import platform\r
 181 \r
 182         try:\r
 183             user = os.getlogin()\r
 184         except:\r
 185             try:\r
 186                 user = os.getenv('USERNAME')\r
 187                 if user is None:\r
 188                     user = os.getenv('USER')\r
 189             except:\r
 190                 user = 'unknown_user'\r
 191         try:\r
 192             (sysname, nodename, release, version, machine,\r
 193              processor) = platform.uname()\r
 194         except:\r
 195             (sysname, nodename, release, version, machine, processor) = (\r
 196                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 197                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 198 \r
 199         self.write_key(write_dict, 'process', None, 'start')\r
 200         write_dict['indentationLevel'] += 1\r
 201 \r
 202         self.write_key(write_dict, 'description', self.description)\r
 203         self.write_key(write_dict, 'cmd', self.cmd)\r
 204         if self.args:\r
 205             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 206         self.write_key(write_dict, 'start', self.start)\r
 207         self.write_key(write_dict, 'end', self.end)\r
 208         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 209 \r
 210         self.write_key(write_dict, 'user', user)\r
 211         self.write_key(write_dict, 'sysname', sysname)\r
 212         self.write_key(write_dict, 'nodename', nodename)\r
 213         self.write_key(write_dict, 'release', release)\r
 214         self.write_key(write_dict, 'version', version)\r
 215         self.write_key(write_dict, 'machine', machine)\r
 216         self.write_key(write_dict, 'processor', processor)\r
 217 \r
 218         if len(self.process_keys) > 0:\r
 219             self.write_key(write_dict, 'processKeys', None, 'start')\r
 220             for pair in self.process_keys:\r
 221                 (key, value) = pair\r
 222                 write_dict['indentationLevel'] += 1\r
 223                 self.write_key(write_dict, key, value)\r
 224                 write_dict['indentationLevel'] -= 1\r
 225             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 226 \r
 227         self.write_key(write_dict, 'status', self.status)\r
 228 \r
 229     def write_log_footer(self, write_dict):\r
 230         """\r
 231         Object description.\r
 232 \r
 233         Parameters\r
 234         ----------\r
 235         parameter : type\r
 236             Parameter description.\r
 237 \r
 238         Returns\r
 239         -------\r
 240         type\r
 241              Return value description.\r
 242         """\r
 243 \r
 244         write_dict['indentationLevel'] -= 1\r
 245         self.write_key(write_dict, 'process', None, 'stop')\r
 246 \r
 247     def write_log(self,\r
 248                   log_handle=sys.stdout,\r
 249                   indentation_level=0,\r
 250                   format='xml'):\r
 251         """\r
 252         Writes logging information to the specified handle.\r
 253 \r
 254         Parameters\r
 255         ----------\r
 256         parameter : type\r
 257             Parameter description.\r
 258 \r
 259         Returns\r
 260         -------\r
 261         type\r
 262              Return value description.\r
 263         """\r
 264 \r
 265         write_dict = {}\r
 266         write_dict['logHandle'] = log_handle\r
 267         write_dict['indentationLevel'] = indentation_level\r
 268         write_dict['format'] = format\r
 269 \r
 270         if log_handle:\r
 271             self.write_log_header(write_dict)\r
 272 \r
 273             if self.log:\r
 274                 self.write_key(write_dict, 'output', None, 'start')\r
 275                 if format == 'xml':\r
 276                     log_handle.write('<![CDATA[\n')\r
 277                 for line in self.log:\r
 278                     log_handle.write('%s%s\n' % ('', line))\r
 279                 if format == 'xml':\r
 280                     log_handle.write(']]>\n')\r
 281                 self.write_key(write_dict, 'output', None, 'stop')\r
 282 \r
 283             self.write_log_footer(write_dict)\r
 284 \r
 285     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 286         """\r
 287         Object description.\r
 288 \r
 289         Parameters\r
 290         ----------\r
 291         parameter : type\r
 292             Parameter description.\r
 293 \r
 294         Returns\r
 295         -------\r
 296         type\r
 297              Return value description.\r
 298         """\r
 299 \r
 300         if log_filename:\r
 301             try:\r
 302                 # TODO: Review statements.\r
 303                 # 3.1\r
 304                 try:\r
 305                     log_handle = (\r
 306                         open(log_filename, mode='wt', encoding='utf-8'))\r
 307                 # 2.6\r
 308                 except:\r
 309                     log_handle = open(log_filename, mode='wt')\r
 310             except:\r
 311                 print('Couldn\'t open log : %s' % log_filename)\r
 312                 log_handle = None\r
 313 \r
 314         if log_handle:\r
 315             if header:\r
 316                 if format == 'xml':\r
 317                     log_handle.write('<![CDATA[\n')\r
 318                 log_handle.write(header)\r
 319                 if format == 'xml':\r
 320                     log_handle.write(']]>\n')\r
 321             self.write_log(log_handle)\r
 322             log_handle.close()\r
 323 \r
 324     def log_line(self, line):\r
 325         """\r
 326         Adds a line of text to the log.\r
 327 \r
 328         Parameters\r
 329         ----------\r
 330         parameter : type\r
 331             Parameter description.\r
 332 \r
 333         Returns\r
 334         -------\r
 335         type\r
 336              Return value description.\r
 337         """\r
 338 \r
 339         self.log.append(line.rstrip())\r
 340         if self.echo:\r
 341             print('%s' % line.rstrip())\r
 342 \r
 343     def execute(self):\r
 344         """\r
 345         Executes the current process.\r
 346 \r
 347         Parameters\r
 348         ----------\r
 349         parameter : type\r
 350             Parameter description.\r
 351 \r
 352         Returns\r
 353         -------\r
 354         type\r
 355              Return value description.\r
 356         """\r
 357 \r
 358         import datetime\r
 359         import traceback\r
 360 \r
 361         try:\r
 362             import subprocess as sp\r
 363         except:\r
 364             sp = None\r
 365 \r
 366         self.start = datetime.datetime.now()\r
 367 \r
 368         cmdargs = [self.cmd]\r
 369         cmdargs.extend(self.args)\r
 370 \r
 371         if self.echo:\r
 372             if sp:\r
 373                 print(\r
 374                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 375             else:\r
 376                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 377 \r
 378         process = None\r
 379         tmp_wrapper = None\r
 380         stdout = None\r
 381         stdin = None\r
 382         parentenv = os.environ\r
 383         parentcwd = os.getcwd()\r
 384 \r
 385         try:\r
 386             # Using *subprocess*.\r
 387             if sp:\r
 388                 if self.batch_wrapper:\r
 389                     cmd = ' '.join(cmdargs)\r
 390                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 391                     write_text(cmd, tmp_wrapper)\r
 392                     print('%s : Running process through wrapper %s\n' % (\r
 393                         self.__class__, tmp_wrapper))\r
 394                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 395                                        stderr=sp.STDOUT,\r
 396                                        cwd=self.cwd, env=self.env)\r
 397                 else:\r
 398                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 399                                        stderr=sp.STDOUT,\r
 400                                        cwd=self.cwd, env=self.env)\r
 401 \r
 402             # using *os.popen4*.\r
 403             else:\r
 404                 if self.env:\r
 405                     os.environ = self.env\r
 406                 if self.cwd:\r
 407                     os.chdir(self.cwd)\r
 408 \r
 409                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 410         except:\r
 411             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 412             traceback.print_exc()\r
 413 \r
 414         # Using *subprocess*\r
 415         if sp:\r
 416             if process != None:\r
 417                 # pid = process.pid\r
 418                 # log.logLine('process id %s\n' % pid)\r
 419 \r
 420                 try:\r
 421                     # This is more proper python, and resolves some issues with\r
 422                     # a process ending before all of its output has been\r
 423                     # processed, but it also seems to stall when the read\r
 424                     # buffer is near or over its limit. This happens\r
 425                     # relatively frequently with processes that generate lots\r
 426                     # of print statements.\r
 427                     for line in process.stdout:\r
 428                         self.log_line(line)\r
 429 \r
 430                     # So we go with the, um, uglier option below.\r
 431 \r
 432                     # This is now used to ensure that the process has finished.\r
 433                     line = ''\r
 434                     while line != None and process.poll() is None:\r
 435                         try:\r
 436                             line = process.stdout.readline()\r
 437                         except:\r
 438                             break\r
 439                         # 3.1\r
 440                         try:\r
 441                             self.log_line(str(line, encoding='utf-8'))\r
 442                         # 2.6\r
 443                         except:\r
 444                             self.log_line(line)\r
 445                 except:\r
 446                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 447 \r
 448                 self.status = process.returncode\r
 449 \r
 450                 if self.batch_wrapper and tmp_wrapper:\r
 451                     try:\r
 452                         os.remove(tmp_wrapper)\r
 453                     except:\r
 454                         print(\r
 455                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 456                         traceback.print_exc()\r
 457 \r
 458         # Using *os.popen4*.\r
 459         else:\r
 460             exit_code = -1\r
 461             try:\r
 462                 stdout_lines = stdout.readlines()\r
 463                 exit_code = stdout.close()\r
 464 \r
 465                 stdout.close()\r
 466                 stdin.close()\r
 467 \r
 468                 if self.env:\r
 469                     os.environ = parentenv\r
 470                 if self.cwd:\r
 471                     os.chdir(parentcwd)\r
 472 \r
 473                 if len(stdout_lines) > 0:\r
 474                     for line in stdout_lines:\r
 475                         self.log_line(line)\r
 476 \r
 477                 if not exit_code:\r
 478                     exit_code = 0\r
 479             except:\r
 480                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 481 \r
 482             self.status = exit_code\r
 483 \r
 484         self.end = datetime.datetime.now()\r
 485 \r
 486 \r
 487 class ProcessList(Process):\r
 488     """\r
 489     A list of processes with logged output.\r
 490     """\r
 491 \r
 492     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 493         """\r
 494         Object description.\r
 495 \r
 496         Parameters\r
 497         ----------\r
 498         parameter : type\r
 499             Parameter description.\r
 500 \r
 501         Returns\r
 502         -------\r
 503         type\r
 504              Return value description.\r
 505         """\r
 506 \r
 507         Process.__init__(self, description, None, None, cwd, env)\r
 508         'Initialize the standard class variables'\r
 509         self.processes = []\r
 510         self.blocking = blocking\r
 511 \r
 512     def generate_report(self, write_dict):\r
 513         """\r
 514         Generates a log based on the success of the child processes.\r
 515 \r
 516         Parameters\r
 517         ----------\r
 518         parameter : type\r
 519             Parameter description.\r
 520 \r
 521         Returns\r
 522         -------\r
 523         type\r
 524              Return value description.\r
 525         """\r
 526 \r
 527         if self.processes:\r
 528             _status = True\r
 529             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 530 \r
 531             self.log = []\r
 532 \r
 533             for child in self.processes:\r
 534                 if isinstance(child, ProcessList):\r
 535                     child.generate_report(write_dict)\r
 536 \r
 537                 key = child.description\r
 538                 value = child.status\r
 539                 if write_dict['format'] == 'xml':\r
 540                     child_result = (\r
 541                         '%s<result description=\'%s\'>%s</result>' % (\r
 542                             indent, key, value))\r
 543                 else:\r
 544                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 545                 self.log.append(child_result)\r
 546 \r
 547                 if child.status != 0:\r
 548                     _status = False\r
 549             if not _status:\r
 550                 self.status = -1\r
 551             else:\r
 552                 self.status = 0\r
 553         else:\r
 554             self.log = ['No child processes available to generate a report']\r
 555             self.status = -1\r
 556 \r
 557     def write_log_header(self, write_dict):\r
 558         """\r
 559         Object description.\r
 560 \r
 561         Parameters\r
 562         ----------\r
 563         parameter : type\r
 564             Parameter description.\r
 565 \r
 566         Returns\r
 567         -------\r
 568         type\r
 569              Return value description.\r
 570         """\r
 571 \r
 572         self.write_key(write_dict, 'processList', None, 'start')\r
 573         write_dict['indentationLevel'] += 1\r
 574 \r
 575         self.write_key(write_dict, 'description', self.description)\r
 576         self.write_key(write_dict, 'start', self.start)\r
 577         self.write_key(write_dict, 'end', self.end)\r
 578         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 579 \r
 580         self.generate_report(write_dict)\r
 581 \r
 582         self.write_key(write_dict, 'status', self.status)\r
 583 \r
 584     def write_log_footer(self, write_dict):\r
 585         """\r
 586         Object description.\r
 587 \r
 588         Parameters\r
 589         ----------\r
 590         parameter : type\r
 591             Parameter description.\r
 592 \r
 593         Returns\r
 594         -------\r
 595         type\r
 596              Return value description.\r
 597         """\r
 598 \r
 599         write_dict['indentationLevel'] -= 1\r
 600         self.write_key(write_dict, 'processList', None, 'stop')\r
 601 \r
 602     def write_log(self,\r
 603                   log_handle=sys.stdout,\r
 604                   indentation_level=0,\r
 605                   format='xml'):\r
 606         """\r
 607         Writes logging information to the specified handle.\r
 608 \r
 609         Parameters\r
 610         ----------\r
 611         parameter : type\r
 612             Parameter description.\r
 613 \r
 614         Returns\r
 615         -------\r
 616         type\r
 617              Return value description.\r
 618         """\r
 619 \r
 620         write_dict = {}\r
 621         write_dict['logHandle'] = log_handle\r
 622         write_dict['indentationLevel'] = indentation_level\r
 623         write_dict['format'] = format\r
 624 \r
 625         if log_handle:\r
 626             self.write_log_header(write_dict)\r
 627 \r
 628             if self.log:\r
 629                 self.write_key(write_dict, 'output', None, 'start')\r
 630                 for line in self.log:\r
 631                     log_handle.write('%s%s\n' % ('', line))\r
 632                 self.write_key(write_dict, 'output', None, 'stop')\r
 633 \r
 634             if self.processes:\r
 635                 self.write_key(write_dict, 'processes', None, 'start')\r
 636                 for child in self.processes:\r
 637                     child.write_log(log_handle, indentation_level + 1, format)\r
 638                 self.write_key(write_dict, 'processes', None, 'stop')\r
 639 \r
 640             self.write_log_footer(write_dict)\r
 641 \r
 642     def execute(self):\r
 643         """\r
 644         Executes the list of processes.\r
 645 \r
 646         Parameters\r
 647         ----------\r
 648         parameter : type\r
 649             Parameter description.\r
 650 \r
 651         Returns\r
 652         -------\r
 653         type\r
 654              Return value description.\r
 655         """\r
 656 \r
 657         import datetime\r
 658 \r
 659         self.start = datetime.datetime.now()\r
 660 \r
 661         self.status = 0\r
 662         if self.processes:\r
 663             for child in self.processes:\r
 664                 if child:\r
 665                     try:\r
 666                         child.execute()\r
 667                     except:\r
 668                         print('%s : caught exception in child class %s' % (\r
 669                             self.__class__, child.__class__))\r
 670                         traceback.print_exc()\r
 671                         child.status = -1\r
 672 \r
 673                     if self.blocking and child.status != 0:\r
 674                         print('%s : child class %s finished with an error' % (\r
 675                             self.__class__, child.__class__))\r
 676                         self.status = -1\r
 677                         break\r
 678 \r
 679         self.end = datetime.datetime.now()\r
 680 \r
 681 \r
 682 def main():\r
 683     """\r
 684     Object description.\r
 685 \r
 686     Parameters\r
 687     ----------\r
 688     parameter : type\r
 689         Parameter description.\r
 690 \r
 691     Returns\r
 692     -------\r
 693     type\r
 694          Return value description.\r
 695     """\r
 696 \r
 697     import optparse\r
 698 \r
 699     p = optparse.OptionParser(description='A process logging script',\r
 700                               prog='process',\r
 701                               version='process 0.1',\r
 702                               usage=('%prog [options] '\r
 703                                      '[options for the logged process]'))\r
 704     p.add_option('--cmd', '-c', default=None)\r
 705     p.add_option('--log', '-l', default=None)\r
 706 \r
 707     options, arguments = p.parse_args()\r
 708 \r
 709     cmd = options.cmd\r
 710     log_filename = options.log\r
 711 \r
 712     try:\r
 713         args_start = sys.argv.index('--') + 1\r
 714         args = sys.argv[args_start:]\r
 715     except:\r
 716         args_start = len(sys.argv) + 1\r
 717         args = []\r
 718 \r
 719     if cmd is None:\r
 720         print('process: No command specified')\r
 721 \r
 722     # Testing regular logging.\r
 723     process = Process(description='a process', cmd=cmd, args=args)\r
 724 \r
 725     # Testing report generation and writing a log.\r
 726     process_list = ProcessList('a process list')\r
 727     process_list.processes.append(process)\r
 728     process_list.echo = True\r
 729     process_list.execute()\r
 730 \r
 731     process_list.write_log_to_disk(log_filename)\r
 732 \r
 733 \r
 734 if __name__ == '__main__':\r
 735     main()\r