Package madgraph :: Package various :: Module misc
[hide private]
[frames] | no frames]

Source Code for Module madgraph.various.misc

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2009 The MadGraph5_aMC@NLO Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph5_aMC@NLO project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph5_aMC@NLO license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, visit madgraph.phys.ucl.ac.be and amcatnlo.web.cern.ch 
  13  # 
  14  ################################################################################ 
  15   
  16  """A set of functions performing routine administrative I/O tasks.""" 
  17   
  18  import contextlib 
  19  import logging 
  20  import os 
  21  import re 
  22  import signal 
  23  import subprocess 
  24  import sys 
  25  import StringIO 
  26  import sys 
  27  import optparse 
  28  import time 
  29  import shutil 
  30  import traceback 
  31  import gzip as ziplib 
  32   
  33  try: 
  34      # Use in MadGraph 
  35      import madgraph 
  36      from madgraph import MadGraph5Error, InvalidCmd 
  37      import madgraph.iolibs.files as files 
  38  except Exception, error: 
  39      if __debug__: 
  40          print error 
  41      # Use in MadEvent 
  42      import internal as madgraph 
  43      from internal import MadGraph5Error, InvalidCmd 
  44      import internal.files as files 
  45       
  46  logger = logging.getLogger('cmdprint.ext_program') 
  47  logger_stderr = logging.getLogger('madevent.misc') 
  48  pjoin = os.path.join 
49 50 #=============================================================================== 51 # parse_info_str 52 #=============================================================================== 53 -def parse_info_str(fsock):
54 """Parse a newline separated list of "param=value" as a dictionnary 55 """ 56 57 info_dict = {} 58 pattern = re.compile("(?P<name>\w*)\s*=\s*(?P<value>.*)", 59 re.IGNORECASE | re.VERBOSE) 60 for entry in fsock: 61 entry = entry.strip() 62 if len(entry) == 0: continue 63 m = pattern.match(entry) 64 if m is not None: 65 info_dict[m.group('name')] = m.group('value') 66 else: 67 raise IOError, "String %s is not a valid info string" % entry 68 69 return info_dict
70
71 72 #=============================================================================== 73 # mute_logger (designed to be a decorator) 74 #=============================================================================== 75 -def mute_logger(names=['madgraph','ALOHA','cmdprint','madevent'], levels=[50,50,50,50]):
76 """change the logger level and restore those at their initial value at the 77 end of the function decorated.""" 78 def control_logger(f): 79 def restore_old_levels(names, levels): 80 for name, level in zip(names, levels): 81 log_module = logging.getLogger(name) 82 log_module.setLevel(level)
83 84 def f_with_no_logger(self, *args, **opt): 85 old_levels = [] 86 for name, level in zip(names, levels): 87 log_module = logging.getLogger(name) 88 old_levels.append(log_module.level) 89 log_module.setLevel(level) 90 try: 91 out = f(self, *args, **opt) 92 restore_old_levels(names, old_levels) 93 return out 94 except: 95 restore_old_levels(names, old_levels) 96 raise 97 98 return f_with_no_logger 99 return control_logger 100
101 #=============================================================================== 102 # get_pkg_info 103 #=============================================================================== 104 -def get_pkg_info(info_str=None):
105 """Returns the current version information of the MadGraph5_aMC@NLO package, 106 as written in the VERSION text file. If the file cannot be found, 107 a dictionary with empty values is returned. As an option, an info 108 string can be passed to be read instead of the file content. 109 """ 110 111 if info_str is None: 112 info_dict = files.read_from_file(os.path.join(madgraph.__path__[0], 113 "VERSION"), 114 parse_info_str, 115 print_error=False) 116 else: 117 info_dict = parse_info_str(StringIO.StringIO(info_str)) 118 119 return info_dict
120
121 #=============================================================================== 122 # get_time_info 123 #=============================================================================== 124 -def get_time_info():
125 """Returns the present time info for use in MG5 command history header. 126 """ 127 128 creation_time = time.asctime() 129 time_info = {'time': creation_time, 130 'fill': ' ' * (26 - len(creation_time))} 131 132 return time_info
133
134 #=============================================================================== 135 # Find the subdirectory which includes the files ending with a given extension 136 #=============================================================================== 137 -def find_includes_path(start_path, extension):
138 """Browse the subdirectories of the path 'start_path' and returns the first 139 one found which contains at least one file ending with the string extension 140 given in argument.""" 141 142 subdirs=[pjoin(start_path,dir) for dir in os.listdir(start_path)] 143 for subdir in subdirs: 144 if os.path.isfile(subdir): 145 if os.path.basename(subdir).endswith(extension): 146 return start_path 147 elif os.path.isdir(subdir): 148 return find_includes_path(subdir, extension) 149 return None
150
151 #=============================================================================== 152 # find a executable 153 #=============================================================================== 154 -def which(program):
155 def is_exe(fpath): 156 return os.path.exists(fpath) and os.access(\ 157 os.path.realpath(fpath), os.X_OK)
158 159 if not program: 160 return None 161 162 fpath, fname = os.path.split(program) 163 if fpath: 164 if is_exe(program): 165 return program 166 else: 167 for path in os.environ["PATH"].split(os.pathsep): 168 exe_file = os.path.join(path, program) 169 if is_exe(exe_file): 170 return exe_file 171 return None 172
173 -def has_f2py():
174 has_f2py = False 175 if which('f2py'): 176 has_f2py = True 177 elif sys.version_info[1] == 6: 178 if which('f2py-2.6'): 179 has_f2py = True 180 elif which('f2py2.6'): 181 has_f2py = True 182 else: 183 if which('f2py-2.7'): 184 has_f2py = True 185 elif which('f2py2.7'): 186 has_f2py = True 187 return has_f2py
188
189 190 191 #=============================================================================== 192 # find a library 193 #=============================================================================== 194 -def which_lib(lib):
195 def is_lib(fpath): 196 return os.path.exists(fpath) and os.access(fpath, os.R_OK)
197 198 if not lib: 199 return None 200 201 fpath, fname = os.path.split(lib) 202 if fpath: 203 if is_lib(lib): 204 return lib 205 else: 206 locations = sum([os.environ[env_path].split(os.pathsep) for env_path in 207 ["DYLD_LIBRARY_PATH","LD_LIBRARY_PATH","LIBRARY_PATH","PATH"] 208 if env_path in os.environ],[]) 209 for path in locations: 210 lib_file = os.path.join(path, lib) 211 if is_lib(lib_file): 212 return lib_file 213 return None 214
215 #=============================================================================== 216 # Return Nice display for a random variable 217 #=============================================================================== 218 -def nice_representation(var, nb_space=0):
219 """ Return nice information on the current variable """ 220 221 #check which data to put: 222 info = [('type',type(var)),('str', var)] 223 if hasattr(var, 'func_doc'): 224 info.append( ('DOC', var.func_doc) ) 225 if hasattr(var, '__doc__'): 226 info.append( ('DOC', var.__doc__) ) 227 if hasattr(var, '__dict__'): 228 info.append( ('ATTRIBUTE', var.__dict__.keys() )) 229 230 spaces = ' ' * nb_space 231 232 outstr='' 233 for name, value in info: 234 outstr += '%s%3s : %s\n' % (spaces,name, value) 235 236 return outstr
237 238 # 239 # Decorator for re-running a crashing function automatically. 240 # 241 wait_once = False
242 -def multiple_try(nb_try=5, sleep=20):
243 244 def deco_retry(f): 245 def deco_f_retry(*args, **opt): 246 for i in range(nb_try): 247 try: 248 return f(*args, **opt) 249 except KeyboardInterrupt: 250 raise 251 except Exception, error: 252 global wait_once 253 if not wait_once: 254 text = """Start waiting for update. (more info in debug mode)""" 255 logger.info(text) 256 logger_stderr.debug('fail to do %s function with %s args. %s try on a max of %s (%s waiting time)' % 257 (str(f), ', '.join([str(a) for a in args]), i+1, nb_try, sleep * (i+1))) 258 logger_stderr.debug('error is %s' % str(error)) 259 if __debug__: logger_stderr.debug('and occurred at :'+traceback.format_exc()) 260 wait_once = True 261 time.sleep(sleep * (i+1)) 262 263 if __debug__: 264 raise 265 raise error.__class__, '[Fail %i times] \n %s ' % (i+1, error)
266 return deco_f_retry 267 return deco_retry 268
269 #=============================================================================== 270 # Compiler which returns smart output error in case of trouble 271 #=============================================================================== 272 -def compile(arg=[], cwd=None, mode='fortran', job_specs = True, nb_core=1 ,**opt):
273 """compile a given directory""" 274 275 cmd = ['make'] 276 try: 277 if nb_core > 1: 278 cmd.append('-j%s' % nb_core) 279 cmd += arg 280 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 281 stderr=subprocess.STDOUT, cwd=cwd, **opt) 282 (out, err) = p.communicate() 283 except OSError, error: 284 if cwd and not os.path.exists(cwd): 285 raise OSError, 'Directory %s doesn\'t exists. Impossible to run make' % cwd 286 else: 287 error_text = "Impossible to compile %s directory\n" % cwd 288 error_text += "Trying to launch make command returns:\n" 289 error_text += " " + str(error) + "\n" 290 error_text += "In general this means that your computer is not able to compile." 291 if sys.platform == "darwin": 292 error_text += "Note that MacOSX doesn\'t have gmake/gfortan install by default.\n" 293 error_text += "Xcode3 contains those required programs" 294 raise MadGraph5Error, error_text 295 296 if p.returncode: 297 # Check that makefile exists 298 if not cwd: 299 cwd = os.getcwd() 300 all_file = [f.lower() for f in os.listdir(cwd)] 301 if 'makefile' not in all_file: 302 raise OSError, 'no makefile present in %s' % os.path.realpath(cwd) 303 304 if mode == 'fortran' and not (which('g77') or which('gfortran')): 305 error_msg = 'A fortran compiler (g77 or gfortran) is required to create this output.\n' 306 error_msg += 'Please install g77 or gfortran on your computer and retry.' 307 raise MadGraph5Error, error_msg 308 elif mode == 'cpp' and not which('g++'): 309 error_msg ='A C++ compiler (g++) is required to create this output.\n' 310 error_msg += 'Please install g++ (which is part of the gcc package) on your computer and retry.' 311 raise MadGraph5Error, error_msg 312 313 # Check if this is due to the need of gfortran 4.6 for quadruple precision 314 if any(tag.upper() in out.upper() for tag in ['real(kind=16)','real*16', 315 'complex*32']) and mode == 'fortran' and not \ 316 ''.join(get_gfortran_version().split('.')) >= '46': 317 if not which('gfortran'): 318 raise MadGraph5Error, 'The fortran compiler gfortran v4.6 or later '+\ 319 'is required to compile %s.\nPlease install it and retry.'%cwd 320 else: 321 logger_stderr.error('ERROR, you could not compile %s because'%cwd+\ 322 ' your version of gfortran is older than 4.6. MadGraph5_aMC@NLO will carry on,'+\ 323 ' but will not be able to compile an executable.') 324 return p.returncode 325 # Other reason 326 error_text = 'A compilation Error occurs ' 327 if cwd: 328 error_text += 'when trying to compile %s.\n' % cwd 329 error_text += 'The compilation fails with the following output message:\n' 330 error_text += ' '+out.replace('\n','\n ')+'\n' 331 error_text += 'Please try to fix this compilations issue and retry.\n' 332 error_text += 'Help might be found at https://answers.launchpad.net/madgraph5.\n' 333 error_text += 'If you think that this is a bug, you can report this at https://bugs.launchpad.net/madgraph5' 334 raise MadGraph5Error, error_text 335 return p.returncode
336
337 -def get_gfortran_version(compiler='gfortran'):
338 """ Returns the gfortran version as a string. 339 Returns '0' if it failed.""" 340 try: 341 p = Popen([compiler, '-dumpversion'], stdout=subprocess.PIPE, 342 stderr=subprocess.PIPE) 343 output, error = p.communicate() 344 version_finder=re.compile(r"(?P<version>(\d.)*\d)") 345 version = version_finder.search(output).group('version') 346 return version 347 except Exception: 348 return '0'
349
350 -def mod_compilator(directory, new='gfortran', current=None, compiler_type='gfortran'):
351 #define global regular expression 352 if type(directory)!=list: 353 directory=[directory] 354 355 #search file 356 file_to_change=find_makefile_in_dir(directory) 357 if compiler_type == 'gfortran': 358 comp_re = re.compile('^(\s*)FC\s*=\s*(.+)\s*$') 359 var = 'FC' 360 elif compiler_type == 'cpp': 361 comp_re = re.compile('^(\s*)CXX\s*=\s*(.+)\s*$') 362 var = 'CXX' 363 else: 364 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 365 366 mod = False 367 for name in file_to_change: 368 lines = open(name,'r').read().split('\n') 369 for iline, line in enumerate(lines): 370 result = comp_re.match(line) 371 if result: 372 if new != result.group(2): 373 mod = True 374 lines[iline] = result.group(1) + var + "=" + new 375 if mod: 376 open(name,'w').write('\n'.join(lines)) 377 # reset it to change the next file 378 mod = False
379
380 #=============================================================================== 381 # mute_logger (designed to work as with statement) 382 #=============================================================================== 383 -class MuteLogger(object):
384 """mute_logger (designed to work as with statement), 385 files allow to redirect the output of the log to a given file. 386 """ 387
388 - def __init__(self, names, levels, files=None, **opt):
389 assert isinstance(names, list) 390 assert isinstance(names, list) 391 392 self.names = names 393 self.levels = levels 394 if isinstance(files, list): 395 self.files = files 396 else: 397 self.files = [files] * len(names) 398 self.logger_saved_info = {} 399 self.opts = opt
400
401 - def __enter__(self):
402 old_levels = [] 403 for name, level, path in zip(self.names, self.levels, self.files): 404 if path: 405 self.setup_logFile_for_logger(path, name, **self.opts) 406 log_module = logging.getLogger(name) 407 old_levels.append(log_module.level) 408 log_module = logging.getLogger(name) 409 log_module.setLevel(level) 410 self.levels = old_levels
411
412 - def __exit__(self, ctype, value, traceback ):
413 for name, level, path, level in zip(self.names, self.levels, self.files, self.levels): 414 if 'keep' in self.opts and not self.opts['keep']: 415 self.restore_logFile_for_logger(name, level, path=path) 416 else: 417 self.restore_logFile_for_logger(name, level) 418 419 log_module = logging.getLogger(name) 420 log_module.setLevel(level)
421
422 - def setup_logFile_for_logger(self, path, full_logname, **opts):
423 """ Setup the logger by redirecting them all to logfiles in tmp """ 424 425 logs = full_logname.split('.') 426 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 427 range(len(full_logname.split('.')))] 428 for logname in lognames: 429 try: 430 os.remove(path) 431 except Exception, error: 432 pass 433 my_logger = logging.getLogger(logname) 434 hdlr = logging.FileHandler(path) 435 # I assume below that the orders of the handlers in my_logger.handlers 436 # remains the same after having added/removed the FileHandler 437 self.logger_saved_info[logname] = [hdlr, my_logger.handlers] 438 #for h in my_logger.handlers: 439 # h.setLevel(logging.CRITICAL) 440 for old_hdlr in list(my_logger.handlers): 441 my_logger.removeHandler(old_hdlr) 442 my_logger.addHandler(hdlr) 443 #my_logger.setLevel(level) 444 my_logger.debug('Log of %s' % logname)
445
446 - def restore_logFile_for_logger(self, full_logname, level, path=None, **opts):
447 """ Setup the logger by redirecting them all to logfiles in tmp """ 448 449 logs = full_logname.split('.') 450 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 451 range(len(full_logname.split('.')))] 452 for logname in lognames: 453 if path: 454 try: 455 os.remove(path) 456 except Exception, error: 457 pass 458 my_logger = logging.getLogger(logname) 459 if logname in self.logger_saved_info: 460 my_logger.removeHandler(self.logger_saved_info[logname][0]) 461 for old_hdlr in self.logger_saved_info[logname][1]: 462 my_logger.addHandler(old_hdlr) 463 else: 464 my_logger.setLevel(level)
465 466 #for i, h in enumerate(my_logger.handlers): 467 # h.setLevel(cls.logger_saved_info[logname][2][i]) 468 469 nb_open =0
470 @contextlib.contextmanager 471 -def stdchannel_redirected(stdchannel, dest_filename):
472 """ 473 A context manager to temporarily redirect stdout or stderr 474 475 e.g.: 476 477 478 with stdchannel_redirected(sys.stderr, os.devnull): 479 if compiler.has_function('clock_gettime', libraries=['rt']): 480 libraries.append('rt') 481 """ 482 483 try: 484 oldstdchannel = os.dup(stdchannel.fileno()) 485 dest_file = open(dest_filename, 'w') 486 os.dup2(dest_file.fileno(), stdchannel.fileno()) 487 yield 488 finally: 489 if oldstdchannel is not None: 490 os.dup2(oldstdchannel, stdchannel.fileno()) 491 os.close(oldstdchannel) 492 if dest_file is not None: 493 dest_file.close()
494
495 -def get_open_fds():
496 ''' 497 return the number of open file descriptors for current process 498 499 .. warning: will only work on UNIX-like os-es. 500 ''' 501 import subprocess 502 import os 503 504 pid = os.getpid() 505 procs = subprocess.check_output( 506 [ "lsof", '-w', '-Ff', "-p", str( pid ) ] ) 507 nprocs = filter( 508 lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(), 509 procs.split( '\n' ) ) 510 511 return nprocs
512
513 -def detect_current_compiler(path, compiler_type='fortran'):
514 """find the current compiler for the current directory""" 515 516 # comp = re.compile("^\s*FC\s*=\s*(\w+)\s*") 517 # The regular expression below allows for compiler definition with absolute path 518 if compiler_type == 'fortran': 519 comp = re.compile("^\s*FC\s*=\s*([\w\/\\.\-]+)\s*") 520 elif compiler_type == 'cpp': 521 comp = re.compile("^\s*CXX\s*=\s*([\w\/\\.\-]+)\s*") 522 else: 523 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 524 525 for line in open(path): 526 if comp.search(line): 527 compiler = comp.search(line).groups()[0] 528 return compiler
529
530 -def find_makefile_in_dir(directory):
531 """ return a list of all file starting with makefile in the given directory""" 532 533 out=[] 534 #list mode 535 if type(directory)==list: 536 for name in directory: 537 out+=find_makefile_in_dir(name) 538 return out 539 540 #single mode 541 for name in os.listdir(directory): 542 if os.path.isdir(directory+'/'+name): 543 out+=find_makefile_in_dir(directory+'/'+name) 544 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('makefile'): 545 out.append(directory+'/'+name) 546 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('make_opt'): 547 out.append(directory+'/'+name) 548 return out
549
550 -def rm_old_compile_file():
551 552 # remove all the .o files 553 os.path.walk('.', rm_file_extension, '.o') 554 555 # remove related libraries 556 libraries = ['libblocks.a', 'libgeneric_mw.a', 'libMWPS.a', 'libtools.a', 'libdhelas3.a', 557 'libdsample.a', 'libgeneric.a', 'libmodel.a', 'libpdf.a', 'libdhelas3.so', 'libTF.a', 558 'libdsample.so', 'libgeneric.so', 'libmodel.so', 'libpdf.so'] 559 lib_pos='./lib' 560 [os.remove(os.path.join(lib_pos, lib)) for lib in libraries \ 561 if os.path.exists(os.path.join(lib_pos, lib))]
562
563 564 -def format_time(n_secs):
565 m, s = divmod(n_secs, 60) 566 h, m = divmod(m, 60) 567 d, h = divmod(h, 24) 568 if d > 0: 569 return "%d day%s,%dh%02dm%02ds" % (d,'' if d<=1 else 's',h, m, s) 570 elif h > 0: 571 return "%dh%02dm%02ds" % (h, m, s) 572 elif m > 0: 573 return "%dm%02ds" % (m, s) 574 else: 575 return "%d second%s" % (s, '' if s<=1 else 's')
576
577 -def rm_file_extension( ext, dirname, names):
578 579 [os.remove(os.path.join(dirname, name)) for name in names if name.endswith(ext)]
580
581 582 583 -def multiple_replacer(*key_values):
584 replace_dict = dict(key_values) 585 replacement_function = lambda match: replace_dict[match.group(0)] 586 pattern = re.compile("|".join([re.escape(k) for k, v in key_values]), re.M) 587 return lambda string: pattern.sub(replacement_function, string)
588
589 -def multiple_replace(string, *key_values):
590 return multiple_replacer(*key_values)(string)
591
592 # Control 593 -def check_system_error(value=1):
594 def deco_check(f): 595 def deco_f(arg, *args, **opt): 596 try: 597 return f(arg, *args, **opt) 598 except OSError, error: 599 logger.debug('try to recover from %s' % error) 600 if isinstance(arg, list): 601 prog = arg[0] 602 else: 603 prog = arg[0] 604 605 # Permission denied 606 if error.errno == 13: 607 if os.path.exists(prog): 608 os.system('chmod +x %s' % prog) 609 elif 'cwd' in opt and opt['cwd'] and \ 610 os.path.isfile(pjoin(opt['cwd'],arg[0])): 611 os.system('chmod +x %s' % pjoin(opt['cwd'],arg[0])) 612 return f(arg, *args, **opt) 613 # NO such file or directory 614 elif error.errno == 2: 615 # raise a more meaningfull error message 616 raise Exception, '%s fails with no such file or directory' \ 617 % arg 618 else: 619 raise
620 return deco_f 621 return deco_check 622
623 624 @check_system_error() 625 -def call(arg, *args, **opt):
626 """nice way to call an external program with nice error treatment""" 627 try: 628 return subprocess.call(arg, *args, **opt) 629 except OSError: 630 arg[0] = './%s' % arg[0] 631 return subprocess.call(arg, *args, **opt)
632
633 @check_system_error() 634 -def Popen(arg, *args, **opt):
635 """nice way to call an external program with nice error treatment""" 636 return subprocess.Popen(arg, *args, **opt)
637
638 @multiple_try() 639 -def mult_try_open(filepath, *args, **opt):
640 """try to open a file with multiple try to ensure that filesystem is sync""" 641 return open(filepath, *args, ** opt)
642
643 644 ################################################################################ 645 # TAIL FUNCTION 646 ################################################################################ 647 -def tail(f, n, offset=None):
648 """Reads a n lines from f with an offset of offset lines. The return 649 value is a tuple in the form ``lines``. 650 """ 651 avg_line_length = 74 652 to_read = n + (offset or 0) 653 654 while 1: 655 try: 656 f.seek(-(avg_line_length * to_read), 2) 657 except IOError: 658 # woops. apparently file is smaller than what we want 659 # to step back, go to the beginning instead 660 f.seek(0) 661 pos = f.tell() 662 lines = f.read().splitlines() 663 if len(lines) >= to_read or pos == 0: 664 return lines[-to_read:offset and -offset or None] 665 avg_line_length *= 1.3 666 avg_line_length = int(avg_line_length)
667
668 ################################################################################ 669 # LAST LINE FUNCTION 670 ################################################################################ 671 -def get_last_line(fsock):
672 """return the last line of a file""" 673 674 return tail(fsock, 1)[0]
675
676 -class BackRead(file):
677 """read a file returning the lines in reverse order for each call of readline() 678 This actually just reads blocks (4096 bytes by default) of data from the end of 679 the file and returns last line in an internal buffer.""" 680 681
682 - def readline(self):
683 """ readline in a backward way """ 684 685 while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): 686 self.blkcount = self.blkcount + 1 687 line = self.data[0] 688 try: 689 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 690 self.data = (self.read(self.blksize) + line).split('\n') 691 except IOError: # can't seek before the beginning of the file 692 self.seek(0) 693 data = self.read(self.size - (self.blksize * (self.blkcount-1))) + line 694 self.data = data.split('\n') 695 696 if len(self.data) == 0: 697 return "" 698 699 line = self.data.pop() 700 return line + '\n'
701
702 - def __init__(self, filepos, blksize=4096):
703 """initialize the internal structures""" 704 705 # get the file size 706 self.size = os.stat(filepos)[6] 707 # how big of a block to read from the file... 708 self.blksize = blksize 709 # how many blocks we've read 710 self.blkcount = 1 711 file.__init__(self, filepos, 'rb') 712 # if the file is smaller than the blocksize, read a block, 713 # otherwise, read the whole thing... 714 if self.size > self.blksize: 715 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 716 self.data = self.read(self.blksize).split('\n') 717 # strip the last item if it's empty... a byproduct of the last line having 718 # a newline at the end of it 719 if not self.data[-1]: 720 self.data.pop()
721
722 - def next(self):
723 line = self.readline() 724 if line: 725 return line 726 else: 727 raise StopIteration
728
729 730 -def write_PS_input(filePath, PS):
731 """ Write out in file filePath the PS point to be read by the MadLoop.""" 732 try: 733 PSfile = open(filePath, 'w') 734 # Add a newline in the end as the implementation fortran 'read' 735 # command on some OS is problematic if it ends directly with the 736 # floating point number read. 737 738 PSfile.write('\n'.join([' '.join(['%.16E'%pi for pi in p]) \ 739 for p in PS])+'\n') 740 PSfile.close() 741 except Exception: 742 raise MadGraph5Error, 'Could not write out the PS point to file %s.'\ 743 %str(filePath)
744
745 -def format_timer(running_time):
746 """ return a nicely string representing the time elapsed.""" 747 if running_time < 2e-2: 748 running_time = running_time = 'current time: %02dh%02d' % (time.localtime().tm_hour, time.localtime().tm_min) 749 elif running_time < 10: 750 running_time = ' %.2gs ' % running_time 751 elif 60 > running_time >= 10: 752 running_time = ' %.3gs ' % running_time 753 elif 3600 > running_time >= 60: 754 running_time = ' %im %is ' % (running_time // 60, int(running_time % 60)) 755 else: 756 running_time = ' %ih %im ' % (running_time // 3600, (running_time//60 % 60)) 757 return running_time
758
759 760 #=============================================================================== 761 # TMP_directory (designed to work as with statement) 762 #=============================================================================== 763 -class TMP_directory(object):
764 """create a temporary directory and ensure this one to be cleaned. 765 """ 766
767 - def __init__(self, suffix='', prefix='tmp', dir=None):
768 self.nb_try_remove = 0 769 import tempfile 770 self.path = tempfile.mkdtemp(suffix, prefix, dir)
771 772
773 - def __exit__(self, ctype, value, traceback ):
774 #True only for debugging: 775 if False and isinstance(value, Exception): 776 sprint("Directory %s not cleaned. This directory can be removed manually" % self.path) 777 return False 778 try: 779 shutil.rmtree(self.path) 780 except OSError: 781 self.nb_try_remove += 1 782 if self.nb_try_remove < 3: 783 time.sleep(10) 784 self.__exit__(ctype, value, traceback) 785 else: 786 logger.warning("Directory %s not completely cleaned. This directory can be removed manually" % self.path)
787
788 - def __enter__(self):
789 return self.path
790 #
791 # GUNZIP/GZIP 792 # 793 -def gunzip(path, keep=False, stdout=None):
794 """ a standard replacement for os.system('gunzip -f %s.gz ' % event_path)""" 795 796 if not path.endswith(".gz"): 797 if os.path.exists("%s.gz" % path): 798 path = "%s.gz" % path 799 else: 800 raise Exception, "%(path)s does not finish by .gz and the file %(path)s.gz does not exists" %\ 801 {"path": path} 802 803 804 #for large file (>1G) it is faster and safer to use a separate thread 805 if os.path.getsize(path) > 1e8: 806 if stdout: 807 os.system('gunzip -c %s > %s' % (path, stdout)) 808 else: 809 os.system('gunzip %s' % path) 810 return 0 811 812 if not stdout: 813 stdout = path[:-3] 814 try: 815 gfile = ziplib.open(path, "r") 816 except IOError: 817 raise 818 else: 819 try: 820 open(stdout,'w').write(gfile.read()) 821 except IOError: 822 # this means that the file is actually not gzip 823 if stdout == path: 824 return 825 else: 826 files.cp(path, stdout) 827 828 if not keep: 829 os.remove(path) 830 return 0
831
832 -def gzip(path, stdout=None, error=True, forceexternal=False):
833 """ a standard replacement for os.system('gzip %s ' % path)""" 834 835 #for large file (>1G) it is faster and safer to use a separate thread 836 if os.path.getsize(path) > 1e9 or forceexternal: 837 call(['gzip', '-f', path]) 838 if stdout: 839 if not stdout.endswith(".gz"): 840 stdout = "%s.gz" % stdout 841 shutil.move('%s.gz' % path, stdout) 842 return 843 844 if not stdout: 845 stdout = "%s.gz" % path 846 elif not stdout.endswith(".gz"): 847 stdout = "%s.gz" % stdout 848 849 try: 850 ziplib.open(stdout,"w").write(open(path).read()) 851 except OverflowError: 852 gzip(path, stdout, error=error, forceexternal=True) 853 except Exception: 854 if error: 855 raise 856 else: 857 os.remove(path)
858
859 # 860 # Global function to open supported file types 861 # 862 -class open_file(object):
863 """ a convinient class to open a file """ 864 865 web_browser = None 866 eps_viewer = None 867 text_editor = None 868 configured = False 869
870 - def __init__(self, filename):
871 """open a file""" 872 873 # Check that the class is correctly configure 874 if not self.configured: 875 self.configure() 876 877 try: 878 extension = filename.rsplit('.',1)[1] 879 except IndexError: 880 extension = '' 881 882 883 # dispatch method 884 if extension in ['html','htm','php']: 885 self.open_program(self.web_browser, filename, background=True) 886 elif extension in ['ps','eps']: 887 self.open_program(self.eps_viewer, filename, background=True) 888 else: 889 self.open_program(self.text_editor,filename, mac_check=False)
890 # mac_check to False avoid to use open cmd in mac 891 892 @classmethod
893 - def configure(cls, configuration=None):
894 """ configure the way to open the file """ 895 896 cls.configured = True 897 898 # start like this is a configuration for mac 899 cls.configure_mac(configuration) 900 if sys.platform == 'darwin': 901 return # done for MAC 902 903 # on Mac some default (eps/web) might be kept on None. This is not 904 #suitable for LINUX which doesn't have open command. 905 906 # first for eps_viewer 907 if not cls.eps_viewer: 908 cls.eps_viewer = cls.find_valid(['evince','gv', 'ggv'], 'eps viewer') 909 910 # Second for web browser 911 if not cls.web_browser: 912 cls.web_browser = cls.find_valid( 913 ['firefox', 'chrome', 'safari','opera'], 914 'web browser')
915 916 @classmethod
917 - def configure_mac(cls, configuration=None):
918 """ configure the way to open a file for mac """ 919 920 if configuration is None: 921 configuration = {'text_editor': None, 922 'eps_viewer':None, 923 'web_browser':None} 924 925 for key in configuration: 926 if key == 'text_editor': 927 # Treat text editor ONLY text base editor !! 928 if configuration[key]: 929 program = configuration[key].split()[0] 930 if not which(program): 931 logger.warning('Specified text editor %s not valid.' % \ 932 configuration[key]) 933 else: 934 # All is good 935 cls.text_editor = configuration[key] 936 continue 937 #Need to find a valid default 938 if os.environ.has_key('EDITOR'): 939 cls.text_editor = os.environ['EDITOR'] 940 else: 941 cls.text_editor = cls.find_valid( 942 ['vi', 'emacs', 'vim', 'gedit', 'nano'], 943 'text editor') 944 945 elif key == 'eps_viewer': 946 if configuration[key]: 947 cls.eps_viewer = configuration[key] 948 continue 949 # else keep None. For Mac this will use the open command. 950 elif key == 'web_browser': 951 if configuration[key]: 952 cls.web_browser = configuration[key] 953 continue
954 # else keep None. For Mac this will use the open command. 955 956 @staticmethod
957 - def find_valid(possibility, program='program'):
958 """find a valid shell program in the list""" 959 960 for p in possibility: 961 if which(p): 962 logger.info('Using default %s \"%s\". ' % (program, p) + \ 963 'Set another one in ./input/mg5_configuration.txt') 964 return p 965 966 logger.info('No valid %s found. ' % program + \ 967 'Please set in ./input/mg5_configuration.txt') 968 return None
969 970
971 - def open_program(self, program, file_path, mac_check=True, background=False):
972 """ open a file with a given program """ 973 974 if mac_check==True and sys.platform == 'darwin': 975 return self.open_mac_program(program, file_path) 976 977 # Shell program only 978 if program: 979 arguments = program.split() # allow argument in program definition 980 arguments.append(file_path) 981 982 if not background: 983 subprocess.call(arguments) 984 else: 985 import thread 986 thread.start_new_thread(subprocess.call,(arguments,)) 987 else: 988 logger.warning('Not able to open file %s since no program configured.' % file_path + \ 989 'Please set one in ./input/mg5_configuration.txt')
990
991 - def open_mac_program(self, program, file_path):
992 """ open a text with the text editor """ 993 994 if not program: 995 # Ask to mac manager 996 os.system('open %s' % file_path) 997 elif which(program): 998 # shell program 999 arguments = program.split() # Allow argument in program definition 1000 arguments.append(file_path) 1001 subprocess.call(arguments) 1002 else: 1003 # not shell program 1004 os.system('open -a %s %s' % (program, file_path))
1005
1006 -def is_executable(path):
1007 """ check if a path is executable""" 1008 try: 1009 return os.access(path, os.X_OK) 1010 except Exception: 1011 return False
1012
1013 -class OptionParser(optparse.OptionParser):
1014 """Option Peaser which raise an error instead as calling exit""" 1015
1016 - def exit(self, status=0, msg=None):
1017 if msg: 1018 raise InvalidCmd, msg 1019 else: 1020 raise InvalidCmd
1021
1022 -def sprint(*args, **opt):
1023 """Returns the current line number in our program.""" 1024 1025 if not __debug__: 1026 return 1027 1028 import inspect 1029 if opt.has_key('log'): 1030 log = opt['log'] 1031 else: 1032 log = logging.getLogger('madgraph') 1033 if opt.has_key('level'): 1034 level = opt['level'] 1035 else: 1036 level = logging.getLogger('madgraph').level 1037 #print "madgraph level",level 1038 #if level == 20: 1039 # level = 10 #avoid info level 1040 #print "use", level 1041 lineno = inspect.currentframe().f_back.f_lineno 1042 fargs = inspect.getframeinfo(inspect.currentframe().f_back) 1043 filename, lineno = fargs[:2] 1044 #file = inspect.currentframe().f_back.co_filename 1045 #print type(file) 1046 try: 1047 source = inspect.getsourcelines(inspect.currentframe().f_back) 1048 line = source[0][lineno-source[1]] 1049 line = re.findall(r"misc\.sprint\(\s*(.*)\)\s*($|#)", line)[0][0] 1050 if line.startswith("'") and line.endswith("'") and line.count(",") ==0: 1051 line= '' 1052 elif line.startswith("\"") and line.endswith("\"") and line.count(",") ==0: 1053 line= '' 1054 elif line.startswith(("\"","'")) and len(args)==1 and "%" in line: 1055 line= '' 1056 except Exception: 1057 line='' 1058 1059 if line: 1060 intro = ' %s = \033[0m' % line 1061 else: 1062 intro = '' 1063 1064 1065 log.log(level, ' '.join([intro]+[str(a) for a in args]) + \ 1066 ' \033[1;30m[%s at line %s]\033[0m' % (os.path.basename(filename), lineno)) 1067 return
1068
1069 ################################################################################ 1070 # function to check if two float are approximatively equal 1071 ################################################################################ 1072 -def equal(a,b,sig_fig=6, zero_limit=True):
1073 """function to check if two float are approximatively equal""" 1074 import math 1075 1076 if not a or not b: 1077 if zero_limit: 1078 power = sig_fig + 1 1079 else: 1080 return a == b 1081 else: 1082 power = sig_fig - int(math.log10(abs(a))) + 1 1083 1084 return ( a==b or abs(int(a*10**power) - int(b*10**power)) < 10)
1085
1086 ################################################################################ 1087 # class to change directory with the "with statement" 1088 ################################################################################ 1089 -class chdir:
1090 - def __init__(self, newPath):
1091 self.newPath = newPath
1092
1093 - def __enter__(self):
1094 self.savedPath = os.getcwd() 1095 os.chdir(self.newPath)
1096
1097 - def __exit__(self, etype, value, traceback):
1098 os.chdir(self.savedPath)
1099
1100 ################################################################################ 1101 # Timeout FUNCTION 1102 ################################################################################ 1103 1104 -def timeout(func, args=(), kwargs={}, timeout_duration=1, default=None):
1105 '''This function will spwan a thread and run the given function using the args, kwargs and 1106 return the given default value if the timeout_duration is exceeded 1107 ''' 1108 import threading 1109 class InterruptableThread(threading.Thread): 1110 def __init__(self): 1111 threading.Thread.__init__(self) 1112 self.result = default
1113 def run(self): 1114 try: 1115 self.result = func(*args, **kwargs) 1116 except Exception,error: 1117 print error 1118 self.result = default 1119 it = InterruptableThread() 1120 it.start() 1121 it.join(timeout_duration) 1122 return it.result 1123
1124 1125 ################################################################################ 1126 # TAIL FUNCTION 1127 ################################################################################ 1128 -class digest:
1129
1130 - def test_all(self):
1131 try: 1132 return self.test_hashlib() 1133 except Exception: 1134 pass 1135 try: 1136 return self.test_md5() 1137 except Exception: 1138 pass 1139 try: 1140 return self.test_zlib() 1141 except Exception: 1142 pass
1143
1144 - def test_hashlib(self):
1145 import hashlib 1146 def digest(text): 1147 """using mg5 for the hash""" 1148 t = hashlib.md5() 1149 t.update(text) 1150 return t.hexdigest()
1151 return digest
1152
1153 - def test_md5(self):
1154 import md5 1155 def digest(text): 1156 """using mg5 for the hash""" 1157 t = md5.md5() 1158 t.update(text) 1159 return t.hexdigest()
1160 return digest 1161
1162 - def test_zlib(self):
1163 import zlib 1164 def digest(text): 1165 return zlib.adler32(text)
1166 1167 digest = digest().test_all()
1168 1169 #=============================================================================== 1170 # Helper class for timing and RAM flashing of subprocesses. 1171 #=============================================================================== 1172 -class ProcessTimer:
1173 - def __init__(self,*args,**opts):
1174 self.cmd_args = args 1175 self.cmd_opts = opts 1176 self.execution_state = False
1177
1178 - def execute(self):
1179 self.max_vms_memory = 0 1180 self.max_rss_memory = 0 1181 1182 self.t1 = None 1183 self.t0 = time.time() 1184 self.p = subprocess.Popen(*self.cmd_args,**self.cmd_opts) 1185 self.execution_state = True
1186
1187 - def poll(self):
1188 if not self.check_execution_state(): 1189 return False 1190 1191 self.t1 = time.time() 1192 flash = subprocess.Popen("ps -p %i -o rss"%self.p.pid, 1193 shell=True,stdout=subprocess.PIPE) 1194 stdout_list = flash.communicate()[0].split('\n') 1195 rss_memory = int(stdout_list[1]) 1196 # for now we ignore vms 1197 vms_memory = 0 1198 1199 # This is the neat version using psutil 1200 # try: 1201 # pp = psutil.Process(self.p.pid) 1202 # 1203 # # obtain a list of the subprocess and all its descendants 1204 # descendants = list(pp.get_children(recursive=True)) 1205 # descendants = descendants + [pp] 1206 # 1207 # rss_memory = 0 1208 # vms_memory = 0 1209 # 1210 # # calculate and sum up the memory of the subprocess and all its descendants 1211 # for descendant in descendants: 1212 # try: 1213 # mem_info = descendant.get_memory_info() 1214 # 1215 # rss_memory += mem_info[0] 1216 # vms_memory += mem_info[1] 1217 # except psutil.error.NoSuchProcess: 1218 # # sometimes a subprocess descendant will have terminated between the time 1219 # # we obtain a list of descendants, and the time we actually poll this 1220 # # descendant's memory usage. 1221 # pass 1222 # 1223 # except psutil.error.NoSuchProcess: 1224 # return self.check_execution_state() 1225 1226 self.max_vms_memory = max(self.max_vms_memory,vms_memory) 1227 self.max_rss_memory = max(self.max_rss_memory,rss_memory) 1228 1229 return self.check_execution_state()
1230
1231 - def is_running(self):
1232 # Version with psutil 1233 # return psutil.pid_exists(self.p.pid) and self.p.poll() == None 1234 return self.p.poll() == None
1235
1236 - def check_execution_state(self):
1237 if not self.execution_state: 1238 return False 1239 if self.is_running(): 1240 return True 1241 self.executation_state = False 1242 self.t1 = time.time() 1243 return False
1244
1245 - def close(self,kill=False):
1246 1247 if self.p.poll() == None: 1248 if kill: 1249 self.p.kill() 1250 else: 1251 self.p.terminate()
1252 1253 # Again a neater handling with psutil 1254 # try: 1255 # pp = psutil.Process(self.p.pid) 1256 # if kill: 1257 # pp.kill() 1258 # else: 1259 # pp.terminate() 1260 # except psutil.error.NoSuchProcess: 1261 # pass 1262