aces_1.0.0/python/aces_ocio/process.py

   1 #!/usr/bin/env python\r
   2 # -*- coding: utf-8 -*-\r
   3 \r
   4 """\r
   5 A process wrapper class that maintains the text output and execution status of\r
   6 a process or a list of other process wrappers which carry such data.\r
   7 """\r
   8 \r
   9 from __future__ import division\r
  10 \r
  11 import os\r
  12 import sys\r
  13 import traceback\r
  14 \r
  15 __author__ = 'ACES Developers'\r
  16 __copyright__ = 'Copyright (C) 2014 - 2015 - ACES Developers'\r
  17 __license__ = ''\r
  18 __maintainer__ = 'ACES Developers'\r
  19 __email__ = 'aces@oscars.org'\r
  20 __status__ = 'Production'\r
  21 \r
  22 __all__ = ['read_text',\r
  23            'write_text',\r
  24            'Process',\r
  25            'ProcessList',\r
  26            'main']\r
  27 \r
  28 \r
  29 def read_text(text_file):\r
  30     """\r
  31     Object description.\r
  32 \r
  33     Parameters\r
  34     ----------\r
  35     parameter : type\r
  36         Parameter description.\r
  37 \r
  38     Returns\r
  39     -------\r
  40     type\r
  41          Return value description.\r
  42     """\r
  43 \r
  44     # TODO: Investigate if check is needed.\r
  45     if not text_file:\r
  46         return\r
  47 \r
  48     with open(text_file, 'rb') as fp:\r
  49         text = (fp.read())\r
  50 \r
  51     return text\r
  52 \r
  53 \r
  54 def write_text(text, text_file):\r
  55     """\r
  56     Object description.\r
  57 \r
  58     Parameters\r
  59     ----------\r
  60     parameter : type\r
  61         Parameter description.\r
  62 \r
  63     Returns\r
  64     -------\r
  65     type\r
  66          Return value description.\r
  67     """\r
  68 \r
  69     # TODO: Investigate if check is needed.\r
  70     if not text_file:\r
  71         return\r
  72 \r
  73     with open(text_file, 'wb') as fp:\r
  74         fp.write(text)\r
  75 \r
  76     return text\r
  77 \r
  78 \r
  79 class Process:\r
  80     """\r
  81     A process with logged output.\r
  82     """\r
  83 \r
  84     def __init__(self,\r
  85                  description=None,\r
  86                  cmd=None,\r
  87                  args=None,\r
  88                  cwd=None,\r
  89                  env=None,\r
  90                  batch_wrapper=False):\r
  91         """\r
  92         Initialize the standard class variables.\r
  93 \r
  94         Parameters\r
  95         ----------\r
  96         parameter : type\r
  97             Parameter description.\r
  98 \r
  99         Returns\r
 100         -------\r
 101         type\r
 102              Return value description.\r
 103         """\r
 104 \r
 105         if args is None:\r
 106             args = []\r
 107 \r
 108         self.cmd = cmd\r
 109         if not description:\r
 110             self.description = cmd\r
 111         else:\r
 112             self.description = description\r
 113         self.status = None\r
 114         self.args = args\r
 115         self.start = None\r
 116         self.end = None\r
 117         self.log = []\r
 118         self.echo = True\r
 119         self.cwd = cwd\r
 120         self.env = env\r
 121         self.batch_wrapper = batch_wrapper\r
 122         self.process_keys = []\r
 123 \r
 124     def get_elapsed_seconds(self):\r
 125         """\r
 126         Object description.\r
 127 \r
 128         Parameters\r
 129         ----------\r
 130         parameter : type\r
 131             Parameter description.\r
 132 \r
 133         Returns\r
 134         -------\r
 135         type\r
 136              Return value description.\r
 137         """\r
 138 \r
 139         import math\r
 140 \r
 141         if self.end and self.start:\r
 142             delta = (self.end - self.start)\r
 143             formatted = '%s.%s' % (delta.days * 86400 + delta.seconds,\r
 144                                    int(math.floor(delta.microseconds / 1e3)))\r
 145         else:\r
 146             formatted = None\r
 147         return formatted\r
 148 \r
 149     def write_key(self, write_dict, key=None, value=None, start_stop=None):\r
 150         """\r
 151         Writes a key / value pair in a supported format.\r
 152 \r
 153         Parameters\r
 154         ----------\r
 155         parameter : type\r
 156             Parameter description.\r
 157 \r
 158         Returns\r
 159         -------\r
 160         type\r
 161              Return value description.\r
 162         """\r
 163 \r
 164         if key is not None and (value is not None or start_stop is not None):\r
 165             indent = '\t' * write_dict['indentationLevel']\r
 166             if write_dict['format'] == 'xml':\r
 167                 if start_stop == 'start':\r
 168                     write_dict['logHandle'].write('%s<%s>\n' % (indent, key))\r
 169                 elif start_stop == 'stop':\r
 170                     write_dict['logHandle'].write('%s</%s>\n' % (indent, key))\r
 171                 else:\r
 172                     write_dict['logHandle'].write(\r
 173                         '%s<%s>%s</%s>\n' % (indent, key, value, key))\r
 174             else:\r
 175                 write_dict['logHandle'].write(\r
 176                     '%s%40s : %s\n' % (indent, key, value))\r
 177 \r
 178     def write_log_header(self, write_dict):\r
 179         """\r
 180         Object description.\r
 181 \r
 182         Parameters\r
 183         ----------\r
 184         parameter : type\r
 185             Parameter description.\r
 186 \r
 187         Returns\r
 188         -------\r
 189         type\r
 190              Return value description.\r
 191         """\r
 192 \r
 193         import platform\r
 194 \r
 195         try:\r
 196             user = os.getlogin()\r
 197         except:\r
 198             try:\r
 199                 user = os.getenv('USERNAME')\r
 200                 if user is None:\r
 201                     user = os.getenv('USER')\r
 202             except:\r
 203                 user = 'unknown_user'\r
 204         try:\r
 205             (sysname, nodename, release, version, machine,\r
 206              processor) = platform.uname()\r
 207         except:\r
 208             (sysname, nodename, release, version, machine, processor) = (\r
 209                 'unknown_sysname', 'unknown_nodename', 'unknown_release',\r
 210                 'unknown_version', 'unknown_machine', 'unknown_processor')\r
 211 \r
 212         self.write_key(write_dict, 'process', None, 'start')\r
 213         write_dict['indentationLevel'] += 1\r
 214 \r
 215         self.write_key(write_dict, 'description', self.description)\r
 216         self.write_key(write_dict, 'cmd', self.cmd)\r
 217         if self.args:\r
 218             self.write_key(write_dict, 'args', ' '.join(self.args))\r
 219         self.write_key(write_dict, 'start', self.start)\r
 220         self.write_key(write_dict, 'end', self.end)\r
 221         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 222 \r
 223         self.write_key(write_dict, 'user', user)\r
 224         self.write_key(write_dict, 'sysname', sysname)\r
 225         self.write_key(write_dict, 'nodename', nodename)\r
 226         self.write_key(write_dict, 'release', release)\r
 227         self.write_key(write_dict, 'version', version)\r
 228         self.write_key(write_dict, 'machine', machine)\r
 229         self.write_key(write_dict, 'processor', processor)\r
 230 \r
 231         if len(self.process_keys) > 0:\r
 232             self.write_key(write_dict, 'processKeys', None, 'start')\r
 233             for pair in self.process_keys:\r
 234                 (key, value) = pair\r
 235                 write_dict['indentationLevel'] += 1\r
 236                 self.write_key(write_dict, key, value)\r
 237                 write_dict['indentationLevel'] -= 1\r
 238             self.write_key(write_dict, 'processKeys', None, 'stop')\r
 239 \r
 240         self.write_key(write_dict, 'status', self.status)\r
 241 \r
 242     def write_log_footer(self, write_dict):\r
 243         """\r
 244         Object description.\r
 245 \r
 246         Parameters\r
 247         ----------\r
 248         parameter : type\r
 249             Parameter description.\r
 250 \r
 251         Returns\r
 252         -------\r
 253         type\r
 254              Return value description.\r
 255         """\r
 256 \r
 257         write_dict['indentationLevel'] -= 1\r
 258         self.write_key(write_dict, 'process', None, 'stop')\r
 259 \r
 260     def write_log(self,\r
 261                   log_handle=sys.stdout,\r
 262                   indentation_level=0,\r
 263                   format='xml'):\r
 264         """\r
 265         Writes logging information to the specified handle.\r
 266 \r
 267         Parameters\r
 268         ----------\r
 269         parameter : type\r
 270             Parameter description.\r
 271 \r
 272         Returns\r
 273         -------\r
 274         type\r
 275              Return value description.\r
 276         """\r
 277 \r
 278         write_dict = {}\r
 279         write_dict['logHandle'] = log_handle\r
 280         write_dict['indentationLevel'] = indentation_level\r
 281         write_dict['format'] = format\r
 282 \r
 283         if log_handle:\r
 284             self.write_log_header(write_dict)\r
 285 \r
 286             if self.log:\r
 287                 self.write_key(write_dict, 'output', None, 'start')\r
 288                 if format == 'xml':\r
 289                     log_handle.write('<![CDATA[\n')\r
 290                 for line in self.log:\r
 291                     log_handle.write('%s%s\n' % ('', line))\r
 292                 if format == 'xml':\r
 293                     log_handle.write(']]>\n')\r
 294                 self.write_key(write_dict, 'output', None, 'stop')\r
 295 \r
 296             self.write_log_footer(write_dict)\r
 297 \r
 298     def write_log_to_disk(self, log_filename=None, format='xml', header=None):\r
 299         """\r
 300         Object description.\r
 301 \r
 302         Parameters\r
 303         ----------\r
 304         parameter : type\r
 305             Parameter description.\r
 306 \r
 307         Returns\r
 308         -------\r
 309         type\r
 310              Return value description.\r
 311         """\r
 312 \r
 313         if log_filename:\r
 314             try:\r
 315                 # TODO: Review statements.\r
 316                 # 3.1\r
 317                 try:\r
 318                     log_handle = (\r
 319                         open(log_filename, mode='wt', encoding='utf-8'))\r
 320                 # 2.6\r
 321                 except:\r
 322                     log_handle = open(log_filename, mode='wt')\r
 323             except:\r
 324                 print('Couldn\'t open log : %s' % log_filename)\r
 325                 log_handle = None\r
 326 \r
 327         if log_handle:\r
 328             if header:\r
 329                 if format == 'xml':\r
 330                     log_handle.write('<![CDATA[\n')\r
 331                 log_handle.write(header)\r
 332                 if format == 'xml':\r
 333                     log_handle.write(']]>\n')\r
 334             self.write_log(log_handle, format=format)\r
 335             log_handle.close()\r
 336 \r
 337     def log_line(self, line):\r
 338         """\r
 339         Adds a line of text to the log.\r
 340 \r
 341         Parameters\r
 342         ----------\r
 343         parameter : type\r
 344             Parameter description.\r
 345 \r
 346         Returns\r
 347         -------\r
 348         type\r
 349              Return value description.\r
 350         """\r
 351 \r
 352         self.log.append(line.rstrip())\r
 353         if self.echo:\r
 354             print('%s' % line.rstrip())\r
 355 \r
 356     def execute(self):\r
 357         """\r
 358         Executes the current process.\r
 359 \r
 360         Parameters\r
 361         ----------\r
 362         parameter : type\r
 363             Parameter description.\r
 364 \r
 365         Returns\r
 366         -------\r
 367         type\r
 368              Return value description.\r
 369         """\r
 370 \r
 371         import datetime\r
 372         import traceback\r
 373 \r
 374         try:\r
 375             import subprocess as sp\r
 376         except:\r
 377             sp = None\r
 378 \r
 379         self.start = datetime.datetime.now()\r
 380 \r
 381         cmdargs = [self.cmd]\r
 382         cmdargs.extend(self.args)\r
 383 \r
 384         if self.echo:\r
 385             if sp:\r
 386                 print(\r
 387                     '\n%s : %s\n' % (self.__class__, sp.list2cmdline(cmdargs)))\r
 388             else:\r
 389                 print('\n%s : %s\n' % (self.__class__, ' '.join(cmdargs)))\r
 390 \r
 391         process = None\r
 392         tmp_wrapper = None\r
 393         stdout = None\r
 394         stdin = None\r
 395         parentenv = os.environ\r
 396         parentcwd = os.getcwd()\r
 397 \r
 398         try:\r
 399             # Using *subprocess*.\r
 400             if sp:\r
 401                 if self.batch_wrapper:\r
 402                     cmd = ' '.join(cmdargs)\r
 403                     tmp_wrapper = os.path.join(self.cwd, 'process.bat')\r
 404                     write_text(cmd, tmp_wrapper)\r
 405                     print('%s : Running process through wrapper %s\n' % (\r
 406                         self.__class__, tmp_wrapper))\r
 407                     process = sp.Popen([tmp_wrapper], stdout=sp.PIPE,\r
 408                                        stderr=sp.STDOUT,\r
 409                                        cwd=self.cwd, env=self.env)\r
 410                 else:\r
 411                     process = sp.Popen(cmdargs, stdout=sp.PIPE,\r
 412                                        stderr=sp.STDOUT,\r
 413                                        cwd=self.cwd, env=self.env)\r
 414 \r
 415             # using *os.popen4*.\r
 416             else:\r
 417                 if self.env:\r
 418                     os.environ = self.env\r
 419                 if self.cwd:\r
 420                     os.chdir(self.cwd)\r
 421 \r
 422                 stdin, stdout = os.popen4(cmdargs, 'r')\r
 423         except:\r
 424             print('Couldn\'t execute command : %s' % cmdargs[0])\r
 425             traceback.print_exc()\r
 426 \r
 427         # Using *subprocess*\r
 428         if sp:\r
 429             if process is not None:\r
 430                 # pid = process.pid\r
 431                 # log.logLine('process id %s\n' % pid)\r
 432 \r
 433                 try:\r
 434                     # This is more proper python, and resolves some issues with\r
 435                     # a process ending before all of its output has been\r
 436                     # processed, but it also seems to stall when the read\r
 437                     # buffer is near or over its limit. This happens\r
 438                     # relatively frequently with processes that generate lots\r
 439                     # of print statements.\r
 440                     for line in process.stdout:\r
 441                         self.log_line(line)\r
 442 \r
 443                     # So we go with the, um, uglier option below.\r
 444 \r
 445                     # This is now used to ensure that the process has finished.\r
 446                     line = ''\r
 447                     while line is not None and process.poll() is None:\r
 448                         try:\r
 449                             line = process.stdout.readline()\r
 450                         except:\r
 451                             break\r
 452                         # 3.1\r
 453                         try:\r
 454                             # TODO: Investigate previous eroneous statement.\r
 455                             # self.log_line(str(line, encoding='utf-8'))\r
 456                             self.log_line(str(line))\r
 457                         # 2.6\r
 458                         except:\r
 459                             self.log_line(line)\r
 460                 except:\r
 461                     self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 462 \r
 463                 self.status = process.returncode\r
 464 \r
 465                 if self.batch_wrapper and tmp_wrapper:\r
 466                     try:\r
 467                         os.remove(tmp_wrapper)\r
 468                     except:\r
 469                         print(\r
 470                             'Couldn\'t remove temp wrapper : %s' % tmp_wrapper)\r
 471                         traceback.print_exc()\r
 472 \r
 473         # Using *os.popen4*.\r
 474         else:\r
 475             exit_code = -1\r
 476             try:\r
 477                 stdout_lines = stdout.readlines()\r
 478                 # TODO: Investigate if this is the good behavior, close() does\r
 479                 # not return anything / None.\r
 480                 exit_code = stdout.close()\r
 481 \r
 482                 stdout.close()\r
 483                 stdin.close()\r
 484 \r
 485                 if self.env:\r
 486                     os.environ = parentenv\r
 487                 if self.cwd:\r
 488                     os.chdir(parentcwd)\r
 489 \r
 490                 if len(stdout_lines) > 0:\r
 491                     for line in stdout_lines:\r
 492                         self.log_line(line)\r
 493 \r
 494                 if not exit_code:\r
 495                     exit_code = 0\r
 496             except:\r
 497                 self.log_line('Logging error : %s' % sys.exc_info()[0])\r
 498 \r
 499             self.status = exit_code\r
 500 \r
 501         self.end = datetime.datetime.now()\r
 502 \r
 503 \r
 504 class ProcessList(Process):\r
 505     """\r
 506     A list of processes with logged output.\r
 507     """\r
 508 \r
 509     def __init__(self, description, blocking=True, cwd=None, env=None):\r
 510         """\r
 511         Object description.\r
 512 \r
 513         Parameters\r
 514         ----------\r
 515         parameter : type\r
 516             Parameter description.\r
 517 \r
 518         Returns\r
 519         -------\r
 520         type\r
 521              Return value description.\r
 522         """\r
 523 \r
 524         Process.__init__(self, description, None, None, cwd, env)\r
 525         'Initialize the standard class variables'\r
 526         self.processes = []\r
 527         self.blocking = blocking\r
 528 \r
 529     def generate_report(self, write_dict):\r
 530         """\r
 531         Generates a log based on the success of the child processes.\r
 532 \r
 533         Parameters\r
 534         ----------\r
 535         parameter : type\r
 536             Parameter description.\r
 537 \r
 538         Returns\r
 539         -------\r
 540         type\r
 541              Return value description.\r
 542         """\r
 543 \r
 544         if self.processes:\r
 545             _status = True\r
 546             indent = '\t' * (write_dict['indentationLevel'] + 1)\r
 547 \r
 548             self.log = []\r
 549 \r
 550             for child in self.processes:\r
 551                 if isinstance(child, ProcessList):\r
 552                     child.generate_report(write_dict)\r
 553 \r
 554                 key = child.description\r
 555                 value = child.status\r
 556                 if write_dict['format'] == 'xml':\r
 557                     child_result = (\r
 558                         '%s<result description=\'%s\'>%s</result>' % (\r
 559                             indent, key, value))\r
 560                 else:\r
 561                     child_result = ('%s%40s : %s' % (indent, key, value))\r
 562                 self.log.append(child_result)\r
 563 \r
 564                 if child.status != 0:\r
 565                     _status = False\r
 566             if not _status:\r
 567                 self.status = -1\r
 568             else:\r
 569                 self.status = 0\r
 570         else:\r
 571             self.log = ['No child processes available to generate a report']\r
 572             self.status = -1\r
 573 \r
 574     def write_log_header(self, write_dict):\r
 575         """\r
 576         Object description.\r
 577 \r
 578         Parameters\r
 579         ----------\r
 580         parameter : type\r
 581             Parameter description.\r
 582 \r
 583         Returns\r
 584         -------\r
 585         type\r
 586              Return value description.\r
 587         """\r
 588 \r
 589         self.write_key(write_dict, 'processList', None, 'start')\r
 590         write_dict['indentationLevel'] += 1\r
 591 \r
 592         self.write_key(write_dict, 'description', self.description)\r
 593         self.write_key(write_dict, 'start', self.start)\r
 594         self.write_key(write_dict, 'end', self.end)\r
 595         self.write_key(write_dict, 'elapsed', self.get_elapsed_seconds())\r
 596 \r
 597         self.generate_report(write_dict)\r
 598 \r
 599         self.write_key(write_dict, 'status', self.status)\r
 600 \r
 601     def write_log_footer(self, write_dict):\r
 602         """\r
 603         Object description.\r
 604 \r
 605         Parameters\r
 606         ----------\r
 607         parameter : type\r
 608             Parameter description.\r
 609 \r
 610         Returns\r
 611         -------\r
 612         type\r
 613              Return value description.\r
 614         """\r
 615 \r
 616         write_dict['indentationLevel'] -= 1\r
 617         self.write_key(write_dict, 'processList', None, 'stop')\r
 618 \r
 619     def write_log(self,\r
 620                   log_handle=sys.stdout,\r
 621                   indentation_level=0,\r
 622                   format='xml'):\r
 623         """\r
 624         Writes logging information to the specified handle.\r
 625 \r
 626         Parameters\r
 627         ----------\r
 628         parameter : type\r
 629             Parameter description.\r
 630 \r
 631         Returns\r
 632         -------\r
 633         type\r
 634              Return value description.\r
 635         """\r
 636 \r
 637         write_dict = {}\r
 638         write_dict['logHandle'] = log_handle\r
 639         write_dict['indentationLevel'] = indentation_level\r
 640         write_dict['format'] = format\r
 641 \r
 642         if log_handle:\r
 643             self.write_log_header(write_dict)\r
 644 \r
 645             if self.log:\r
 646                 self.write_key(write_dict, 'output', None, 'start')\r
 647                 for line in self.log:\r
 648                     log_handle.write('%s%s\n' % ('', line))\r
 649                 self.write_key(write_dict, 'output', None, 'stop')\r
 650 \r
 651             if self.processes:\r
 652                 self.write_key(write_dict, 'processes', None, 'start')\r
 653                 for child in self.processes:\r
 654                     child.write_log(log_handle, indentation_level + 1, format)\r
 655                 self.write_key(write_dict, 'processes', None, 'stop')\r
 656 \r
 657             self.write_log_footer(write_dict)\r
 658 \r
 659     def execute(self):\r
 660         """\r
 661         Executes the list of processes.\r
 662 \r
 663         Parameters\r
 664         ----------\r
 665         parameter : type\r
 666             Parameter description.\r
 667 \r
 668         Returns\r
 669         -------\r
 670         type\r
 671              Return value description.\r
 672         """\r
 673 \r
 674         import datetime\r
 675 \r
 676         self.start = datetime.datetime.now()\r
 677 \r
 678         self.status = 0\r
 679         if self.processes:\r
 680             for child in self.processes:\r
 681                 if child:\r
 682                     try:\r
 683                         child.execute()\r
 684                     except:\r
 685                         print('%s : caught exception in child class %s' % (\r
 686                             self.__class__, child.__class__))\r
 687                         traceback.print_exc()\r
 688                         child.status = -1\r
 689 \r
 690                     if self.blocking and child.status != 0:\r
 691                         print('%s : child class %s finished with an error' % (\r
 692                             self.__class__, child.__class__))\r
 693                         self.status = -1\r
 694                         break\r
 695 \r
 696         self.end = datetime.datetime.now()\r
 697 \r
 698 \r
 699 def main():\r
 700     """\r
 701     Object description.\r
 702 \r
 703     Parameters\r
 704     ----------\r
 705     parameter : type\r
 706         Parameter description.\r
 707 \r
 708     Returns\r
 709     -------\r
 710     type\r
 711          Return value description.\r
 712     """\r
 713 \r
 714     import optparse\r
 715 \r
 716     p = optparse.OptionParser(description='A process logging script',\r
 717                               prog='process',\r
 718                               version='process 0.1',\r
 719                               usage=('%prog [options] '\r
 720                                      '[options for the logged process]'))\r
 721     p.add_option('--cmd', '-c', default=None)\r
 722     p.add_option('--log', '-l', default=None)\r
 723 \r
 724     options, arguments = p.parse_args()\r
 725 \r
 726     cmd = options.cmd\r
 727     log_filename = options.log\r
 728 \r
 729     try:\r
 730         args_start = sys.argv.index('--') + 1\r
 731         args = sys.argv[args_start:]\r
 732     except:\r
 733         args = []\r
 734 \r
 735     if cmd is None:\r
 736         print('process: No command specified')\r
 737 \r
 738     # Testing regular logging.\r
 739     process = Process(description='a process', cmd=cmd, args=args)\r
 740 \r
 741     # Testing report generation and writing a log.\r
 742     process_list = ProcessList('a process list')\r
 743     process_list.processes.append(process)\r
 744     process_list.echo = True\r
 745     process_list.execute()\r
 746 \r
 747     process_list.write_log_to_disk(log_filename)\r
 748 \r
 749 \r
 750 if __name__ == '__main__':\r
 751     main()\r