Package madgraph :: Package various :: Module misc
[hide private]
[frames] | no frames]

Source Code for Module madgraph.various.misc

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2009 The MadGraph5_aMC@NLO Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph5_aMC@NLO project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph5_aMC@NLO license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, visit madgraph.phys.ucl.ac.be and amcatnlo.web.cern.ch 
  13  # 
  14  ################################################################################ 
  15   
  16  """A set of functions performing routine administrative I/O tasks.""" 
  17   
  18  import contextlib 
  19  import logging 
  20  import os 
  21  import re 
  22  import signal 
  23  import subprocess 
  24  import sys 
  25  import StringIO 
  26  import sys 
  27  import optparse 
  28  import time 
  29  import shutil 
  30  import traceback 
  31  import gzip as ziplib 
  32   
  33  try: 
  34      # Use in MadGraph 
  35      import madgraph 
  36  except Exception, error: 
  37      # Use in MadEvent 
  38      import internal 
  39      from internal import MadGraph5Error, InvalidCmd 
  40      import internal.files as files 
  41      MADEVENT = True     
  42  else: 
  43      from madgraph import MadGraph5Error, InvalidCmd 
  44      import madgraph.iolibs.files as files 
  45      MADEVENT = False 
  46   
  47   
  48       
  49  logger = logging.getLogger('cmdprint.ext_program') 
  50  logger_stderr = logging.getLogger('madevent.misc') 
  51  pjoin = os.path.join 
52 53 #=============================================================================== 54 # parse_info_str 55 #=============================================================================== 56 -def parse_info_str(fsock):
57 """Parse a newline separated list of "param=value" as a dictionnary 58 """ 59 60 info_dict = {} 61 pattern = re.compile("(?P<name>\w*)\s*=\s*(?P<value>.*)", 62 re.IGNORECASE | re.VERBOSE) 63 for entry in fsock: 64 entry = entry.strip() 65 if len(entry) == 0: continue 66 m = pattern.match(entry) 67 if m is not None: 68 info_dict[m.group('name')] = m.group('value') 69 else: 70 raise IOError, "String %s is not a valid info string" % entry 71 72 return info_dict
73
74 75 #=============================================================================== 76 # mute_logger (designed to be a decorator) 77 #=============================================================================== 78 -def mute_logger(names=['madgraph','ALOHA','cmdprint','madevent'], levels=[50,50,50,50]):
79 """change the logger level and restore those at their initial value at the 80 end of the function decorated.""" 81 def control_logger(f): 82 def restore_old_levels(names, levels): 83 for name, level in zip(names, levels): 84 log_module = logging.getLogger(name) 85 log_module.setLevel(level)
86 87 def f_with_no_logger(self, *args, **opt): 88 old_levels = [] 89 for name, level in zip(names, levels): 90 log_module = logging.getLogger(name) 91 old_levels.append(log_module.level) 92 log_module.setLevel(level) 93 try: 94 out = f(self, *args, **opt) 95 restore_old_levels(names, old_levels) 96 return out 97 except: 98 restore_old_levels(names, old_levels) 99 raise 100 101 return f_with_no_logger 102 return control_logger 103
104 #=============================================================================== 105 # get_pkg_info 106 #=============================================================================== 107 -def get_pkg_info(info_str=None):
108 """Returns the current version information of the MadGraph5_aMC@NLO package, 109 as written in the VERSION text file. If the file cannot be found, 110 a dictionary with empty values is returned. As an option, an info 111 string can be passed to be read instead of the file content. 112 """ 113 114 if info_str: 115 info_dict = parse_info_str(StringIO.StringIO(info_str)) 116 117 elif MADEVENT: 118 info_dict ={} 119 info_dict['version'] = open(pjoin(internal.__path__[0],'..','..','MGMEVersion.txt')).read().strip() 120 info_dict['date'] = '20xx-xx-xx' 121 else: 122 info_dict = files.read_from_file(os.path.join(madgraph.__path__[0], 123 "VERSION"), 124 parse_info_str, 125 print_error=False) 126 127 return info_dict
128
129 #=============================================================================== 130 # get_time_info 131 #=============================================================================== 132 -def get_time_info():
133 """Returns the present time info for use in MG5 command history header. 134 """ 135 136 creation_time = time.asctime() 137 time_info = {'time': creation_time, 138 'fill': ' ' * (26 - len(creation_time))} 139 140 return time_info
141
142 #=============================================================================== 143 # Find the subdirectory which includes the files ending with a given extension 144 #=============================================================================== 145 -def find_includes_path(start_path, extension):
146 """Browse the subdirectories of the path 'start_path' and returns the first 147 one found which contains at least one file ending with the string extension 148 given in argument.""" 149 150 subdirs=[pjoin(start_path,dir) for dir in os.listdir(start_path)] 151 for subdir in subdirs: 152 if os.path.isfile(subdir): 153 if os.path.basename(subdir).endswith(extension): 154 return start_path 155 elif os.path.isdir(subdir): 156 return find_includes_path(subdir, extension) 157 return None
158
159 #=============================================================================== 160 # find a executable 161 #=============================================================================== 162 -def which(program):
163 def is_exe(fpath): 164 return os.path.exists(fpath) and os.access(\ 165 os.path.realpath(fpath), os.X_OK)
166 167 if not program: 168 return None 169 170 fpath, fname = os.path.split(program) 171 if fpath: 172 if is_exe(program): 173 return program 174 else: 175 for path in os.environ["PATH"].split(os.pathsep): 176 exe_file = os.path.join(path, program) 177 if is_exe(exe_file): 178 return exe_file 179 return None 180
181 -def has_f2py():
182 has_f2py = False 183 if which('f2py'): 184 has_f2py = True 185 elif sys.version_info[1] == 6: 186 if which('f2py-2.6'): 187 has_f2py = True 188 elif which('f2py2.6'): 189 has_f2py = True 190 else: 191 if which('f2py-2.7'): 192 has_f2py = True 193 elif which('f2py2.7'): 194 has_f2py = True 195 return has_f2py
196
197 #=============================================================================== 198 # Activate dependencies if possible. Mainly for tests 199 #=============================================================================== 200 201 -def deactivate_dependence(dependency, cmd=None, log = None):
202 """ Make sure to turn off some dependency of MG5aMC. """ 203 204 def tell(msg): 205 if log == 'stdout': 206 print msg 207 elif callable(log): 208 log(msg)
209 210 211 if dependency in ['pjfry','golem']: 212 if cmd.options[dependency] not in ['None',None,'']: 213 tell("Deactivating MG5_aMC dependency '%s'"%dependency) 214 cmd.options[dependency] = 'None' 215
216 -def activate_dependence(dependency, cmd=None, log = None):
217 """ Checks whether the specfieid MG dependency can be activated if it was 218 not turned off in MG5 options.""" 219 220 def tell(msg): 221 if log == 'stdout': 222 print msg 223 elif callable(log): 224 log(msg)
225 226 if cmd is None: 227 cmd = MGCmd.MasterCmd() 228 229 if dependency=='pjfry': 230 if cmd.options['pjfry'] in ['None',None,''] or \ 231 (cmd.options['pjfry'] == 'auto' and which_lib('libpjfry.a') is None) or\ 232 which_lib(pjoin(cmd.options['pjfry'],'libpjfry.a')) is None: 233 tell("Installing PJFry...") 234 cmd.do_install('PJFry') 235 236 if dependency=='golem': 237 if cmd.options['golem'] in ['None',None,''] or\ 238 (cmd.options['golem'] == 'auto' and which_lib('libgolem.a') is None) or\ 239 which_lib(pjoin(cmd.options['golem'],'libgolem.a')) is None: 240 tell("Installing Golem95...") 241 cmd.do_install('Golem95') 242
243 #=============================================================================== 244 # find a library 245 #=============================================================================== 246 -def which_lib(lib):
247 def is_lib(fpath): 248 return os.path.exists(fpath) and os.access(fpath, os.R_OK)
249 250 if not lib: 251 return None 252 253 fpath, fname = os.path.split(lib) 254 if fpath: 255 if is_lib(lib): 256 return lib 257 else: 258 locations = sum([os.environ[env_path].split(os.pathsep) for env_path in 259 ["DYLD_LIBRARY_PATH","LD_LIBRARY_PATH","LIBRARY_PATH","PATH"] 260 if env_path in os.environ],[]) 261 for path in locations: 262 lib_file = os.path.join(path, lib) 263 if is_lib(lib_file): 264 return lib_file 265 return None 266
267 #=============================================================================== 268 # Return Nice display for a random variable 269 #=============================================================================== 270 -def nice_representation(var, nb_space=0):
271 """ Return nice information on the current variable """ 272 273 #check which data to put: 274 info = [('type',type(var)),('str', var)] 275 if hasattr(var, 'func_doc'): 276 info.append( ('DOC', var.func_doc) ) 277 if hasattr(var, '__doc__'): 278 info.append( ('DOC', var.__doc__) ) 279 if hasattr(var, '__dict__'): 280 info.append( ('ATTRIBUTE', var.__dict__.keys() )) 281 282 spaces = ' ' * nb_space 283 284 outstr='' 285 for name, value in info: 286 outstr += '%s%3s : %s\n' % (spaces,name, value) 287 288 return outstr
289 290 # 291 # Decorator for re-running a crashing function automatically. 292 # 293 wait_once = False
294 -def multiple_try(nb_try=5, sleep=20):
295 296 def deco_retry(f): 297 def deco_f_retry(*args, **opt): 298 for i in range(nb_try): 299 try: 300 return f(*args, **opt) 301 except KeyboardInterrupt: 302 raise 303 except Exception, error: 304 global wait_once 305 if not wait_once: 306 text = """Start waiting for update. (more info in debug mode)""" 307 logger.info(text) 308 logger_stderr.debug('fail to do %s function with %s args. %s try on a max of %s (%s waiting time)' % 309 (str(f), ', '.join([str(a) for a in args]), i+1, nb_try, sleep * (i+1))) 310 logger_stderr.debug('error is %s' % str(error)) 311 if __debug__: logger_stderr.debug('and occurred at :'+traceback.format_exc()) 312 wait_once = True 313 time.sleep(sleep * (i+1)) 314 315 if __debug__: 316 raise 317 raise error.__class__, '[Fail %i times] \n %s ' % (i+1, error)
318 return deco_f_retry 319 return deco_retry 320
321 #=============================================================================== 322 # helper for scan. providing a nice formatted string for the scan name 323 #=============================================================================== 324 -def get_scan_name(first, last):
325 """return a name of the type xxxx[A-B]yyy 326 where xxx and yyy are the common part between the two names. 327 """ 328 329 # find the common string at the beginning 330 base = [first[i] for i in range(len(first)) if first[:i+1] == last[:i+1]] 331 # remove digit even if in common 332 while base[-1].isdigit(): 333 base = base[:-1] 334 # find the common string at the end 335 end = [first[-(i+1)] for i in range(len(first)) if first[-(i+1):] == last[-(i+1):]] 336 # remove digit even if in common 337 while end and end[-1].isdigit(): 338 end = end[:-1] 339 end.reverse() 340 #convert to string 341 base, end = ''.join(base), ''.join(end) 342 if end: 343 name = "%s[%s-%s]%s" % (base, first[len(base):-len(end)], last[len(base):-len(end)],end) 344 else: 345 name = "%s[%s-%s]%s" % (base, first[len(base):], last[len(base):],end) 346 return name
347
348 #=============================================================================== 349 # Compiler which returns smart output error in case of trouble 350 #=============================================================================== 351 -def compile(arg=[], cwd=None, mode='fortran', job_specs = True, nb_core=1 ,**opt):
352 """compile a given directory""" 353 354 cmd = ['make'] 355 try: 356 if nb_core > 1: 357 cmd.append('-j%s' % nb_core) 358 cmd += arg 359 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 360 stderr=subprocess.STDOUT, cwd=cwd, **opt) 361 (out, err) = p.communicate() 362 except OSError, error: 363 if cwd and not os.path.exists(cwd): 364 raise OSError, 'Directory %s doesn\'t exists. Impossible to run make' % cwd 365 else: 366 error_text = "Impossible to compile %s directory\n" % cwd 367 error_text += "Trying to launch make command returns:\n" 368 error_text += " " + str(error) + "\n" 369 error_text += "In general this means that your computer is not able to compile." 370 if sys.platform == "darwin": 371 error_text += "Note that MacOSX doesn\'t have gmake/gfortan install by default.\n" 372 error_text += "Xcode3 contains those required programs" 373 raise MadGraph5Error, error_text 374 375 if p.returncode: 376 # Check that makefile exists 377 if not cwd: 378 cwd = os.getcwd() 379 all_file = [f.lower() for f in os.listdir(cwd)] 380 if 'makefile' not in all_file: 381 raise OSError, 'no makefile present in %s' % os.path.realpath(cwd) 382 383 if mode == 'fortran' and not (which('g77') or which('gfortran')): 384 error_msg = 'A fortran compiler (g77 or gfortran) is required to create this output.\n' 385 error_msg += 'Please install g77 or gfortran on your computer and retry.' 386 raise MadGraph5Error, error_msg 387 elif mode == 'cpp' and not which('g++'): 388 error_msg ='A C++ compiler (g++) is required to create this output.\n' 389 error_msg += 'Please install g++ (which is part of the gcc package) on your computer and retry.' 390 raise MadGraph5Error, error_msg 391 392 # Check if this is due to the need of gfortran 4.6 for quadruple precision 393 if any(tag.upper() in out.upper() for tag in ['real(kind=16)','real*16', 394 'complex*32']) and mode == 'fortran' and not \ 395 ''.join(get_gfortran_version().split('.')) >= '46': 396 if not which('gfortran'): 397 raise MadGraph5Error, 'The fortran compiler gfortran v4.6 or later '+\ 398 'is required to compile %s.\nPlease install it and retry.'%cwd 399 else: 400 logger_stderr.error('ERROR, you could not compile %s because'%cwd+\ 401 ' your version of gfortran is older than 4.6. MadGraph5_aMC@NLO will carry on,'+\ 402 ' but will not be able to compile an executable.') 403 return p.returncode 404 # Other reason 405 error_text = 'A compilation Error occurs ' 406 if cwd: 407 error_text += 'when trying to compile %s.\n' % cwd 408 error_text += 'The compilation fails with the following output message:\n' 409 error_text += ' '+out.replace('\n','\n ')+'\n' 410 error_text += 'Please try to fix this compilations issue and retry.\n' 411 error_text += 'Help might be found at https://answers.launchpad.net/madgraph5.\n' 412 error_text += 'If you think that this is a bug, you can report this at https://bugs.launchpad.net/madgraph5' 413 raise MadGraph5Error, error_text 414 return p.returncode
415
416 -def get_gfortran_version(compiler='gfortran'):
417 """ Returns the gfortran version as a string. 418 Returns '0' if it failed.""" 419 try: 420 p = Popen([compiler, '-dumpversion'], stdout=subprocess.PIPE, 421 stderr=subprocess.PIPE) 422 output, error = p.communicate() 423 version_finder=re.compile(r"(?P<version>(\d.)*\d)") 424 version = version_finder.search(output).group('version') 425 return version 426 except Exception: 427 return '0'
428
429 -def mod_compilator(directory, new='gfortran', current=None, compiler_type='gfortran'):
430 #define global regular expression 431 if type(directory)!=list: 432 directory=[directory] 433 434 #search file 435 file_to_change=find_makefile_in_dir(directory) 436 if compiler_type == 'gfortran': 437 comp_re = re.compile('^(\s*)FC\s*=\s*(.+)\s*$') 438 var = 'FC' 439 elif compiler_type == 'cpp': 440 comp_re = re.compile('^(\s*)CXX\s*=\s*(.+)\s*$') 441 var = 'CXX' 442 else: 443 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 444 445 mod = False 446 for name in file_to_change: 447 lines = open(name,'r').read().split('\n') 448 for iline, line in enumerate(lines): 449 result = comp_re.match(line) 450 if result: 451 if new != result.group(2): 452 mod = True 453 lines[iline] = result.group(1) + var + "=" + new 454 if mod: 455 open(name,'w').write('\n'.join(lines)) 456 # reset it to change the next file 457 mod = False
458
459 #=============================================================================== 460 # mute_logger (designed to work as with statement) 461 #=============================================================================== 462 -class MuteLogger(object):
463 """mute_logger (designed to work as with statement), 464 files allow to redirect the output of the log to a given file. 465 """ 466
467 - def __init__(self, names, levels, files=None, **opt):
468 assert isinstance(names, list) 469 assert isinstance(names, list) 470 471 self.names = names 472 self.levels = levels 473 if isinstance(files, list): 474 self.files = files 475 else: 476 self.files = [files] * len(names) 477 self.logger_saved_info = {} 478 self.opts = opt
479
480 - def __enter__(self):
481 old_levels = [] 482 for name, level, path in zip(self.names, self.levels, self.files): 483 if path: 484 self.setup_logFile_for_logger(path, name, **self.opts) 485 log_module = logging.getLogger(name) 486 old_levels.append(log_module.level) 487 log_module = logging.getLogger(name) 488 log_module.setLevel(level) 489 self.levels = old_levels
490
491 - def __exit__(self, ctype, value, traceback ):
492 for name, level, path, level in zip(self.names, self.levels, self.files, self.levels): 493 if 'keep' in self.opts and not self.opts['keep']: 494 self.restore_logFile_for_logger(name, level, path=path) 495 else: 496 self.restore_logFile_for_logger(name, level) 497 498 log_module = logging.getLogger(name) 499 log_module.setLevel(level)
500
501 - def setup_logFile_for_logger(self, path, full_logname, **opts):
502 """ Setup the logger by redirecting them all to logfiles in tmp """ 503 504 logs = full_logname.split('.') 505 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 506 range(len(full_logname.split('.')))] 507 for logname in lognames: 508 try: 509 os.remove(path) 510 except Exception, error: 511 pass 512 my_logger = logging.getLogger(logname) 513 hdlr = logging.FileHandler(path) 514 # I assume below that the orders of the handlers in my_logger.handlers 515 # remains the same after having added/removed the FileHandler 516 self.logger_saved_info[logname] = [hdlr, my_logger.handlers] 517 #for h in my_logger.handlers: 518 # h.setLevel(logging.CRITICAL) 519 for old_hdlr in list(my_logger.handlers): 520 my_logger.removeHandler(old_hdlr) 521 my_logger.addHandler(hdlr) 522 #my_logger.setLevel(level) 523 my_logger.debug('Log of %s' % logname)
524
525 - def restore_logFile_for_logger(self, full_logname, level, path=None, **opts):
526 """ Setup the logger by redirecting them all to logfiles in tmp """ 527 528 logs = full_logname.split('.') 529 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 530 range(len(full_logname.split('.')))] 531 for logname in lognames: 532 if path: 533 try: 534 os.remove(path) 535 except Exception, error: 536 pass 537 my_logger = logging.getLogger(logname) 538 if logname in self.logger_saved_info: 539 my_logger.removeHandler(self.logger_saved_info[logname][0]) 540 for old_hdlr in self.logger_saved_info[logname][1]: 541 my_logger.addHandler(old_hdlr) 542 else: 543 my_logger.setLevel(level)
544 545 #for i, h in enumerate(my_logger.handlers): 546 # h.setLevel(cls.logger_saved_info[logname][2][i]) 547 548 nb_open =0
549 @contextlib.contextmanager 550 -def stdchannel_redirected(stdchannel, dest_filename):
551 """ 552 A context manager to temporarily redirect stdout or stderr 553 554 e.g.: 555 556 557 with stdchannel_redirected(sys.stderr, os.devnull): 558 if compiler.has_function('clock_gettime', libraries=['rt']): 559 libraries.append('rt') 560 """ 561 562 try: 563 oldstdchannel = os.dup(stdchannel.fileno()) 564 dest_file = open(dest_filename, 'w') 565 os.dup2(dest_file.fileno(), stdchannel.fileno()) 566 yield 567 finally: 568 if oldstdchannel is not None: 569 os.dup2(oldstdchannel, stdchannel.fileno()) 570 os.close(oldstdchannel) 571 if dest_file is not None: 572 dest_file.close()
573
574 -def get_open_fds():
575 ''' 576 return the number of open file descriptors for current process 577 578 .. warning: will only work on UNIX-like os-es. 579 ''' 580 import subprocess 581 import os 582 583 pid = os.getpid() 584 procs = subprocess.check_output( 585 [ "lsof", '-w', '-Ff', "-p", str( pid ) ] ) 586 nprocs = filter( 587 lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(), 588 procs.split( '\n' ) ) 589 590 return nprocs
591
592 -def detect_current_compiler(path, compiler_type='fortran'):
593 """find the current compiler for the current directory""" 594 595 # comp = re.compile("^\s*FC\s*=\s*(\w+)\s*") 596 # The regular expression below allows for compiler definition with absolute path 597 if compiler_type == 'fortran': 598 comp = re.compile("^\s*FC\s*=\s*([\w\/\\.\-]+)\s*") 599 elif compiler_type == 'cpp': 600 comp = re.compile("^\s*CXX\s*=\s*([\w\/\\.\-]+)\s*") 601 else: 602 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 603 604 for line in open(path): 605 if comp.search(line): 606 compiler = comp.search(line).groups()[0] 607 return compiler
608
609 -def find_makefile_in_dir(directory):
610 """ return a list of all file starting with makefile in the given directory""" 611 612 out=[] 613 #list mode 614 if type(directory)==list: 615 for name in directory: 616 out+=find_makefile_in_dir(name) 617 return out 618 619 #single mode 620 for name in os.listdir(directory): 621 if os.path.isdir(directory+'/'+name): 622 out+=find_makefile_in_dir(directory+'/'+name) 623 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('makefile'): 624 out.append(directory+'/'+name) 625 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('make_opt'): 626 out.append(directory+'/'+name) 627 return out
628
629 -def rm_old_compile_file():
630 631 # remove all the .o files 632 os.path.walk('.', rm_file_extension, '.o') 633 634 # remove related libraries 635 libraries = ['libblocks.a', 'libgeneric_mw.a', 'libMWPS.a', 'libtools.a', 'libdhelas3.a', 636 'libdsample.a', 'libgeneric.a', 'libmodel.a', 'libpdf.a', 'libdhelas3.so', 'libTF.a', 637 'libdsample.so', 'libgeneric.so', 'libmodel.so', 'libpdf.so'] 638 lib_pos='./lib' 639 [os.remove(os.path.join(lib_pos, lib)) for lib in libraries \ 640 if os.path.exists(os.path.join(lib_pos, lib))]
641
642 643 -def format_time(n_secs):
644 m, s = divmod(n_secs, 60) 645 h, m = divmod(m, 60) 646 d, h = divmod(h, 24) 647 if d > 0: 648 return "%d day%s,%dh%02dm%02ds" % (d,'' if d<=1 else 's',h, m, s) 649 elif h > 0: 650 return "%dh%02dm%02ds" % (h, m, s) 651 elif m > 0: 652 return "%dm%02ds" % (m, s) 653 else: 654 return "%d second%s" % (s, '' if s<=1 else 's')
655
656 -def rm_file_extension( ext, dirname, names):
657 658 [os.remove(os.path.join(dirname, name)) for name in names if name.endswith(ext)]
659
660 661 662 -def multiple_replacer(*key_values):
663 replace_dict = dict(key_values) 664 replacement_function = lambda match: replace_dict[match.group(0)] 665 pattern = re.compile("|".join([re.escape(k) for k, v in key_values]), re.M) 666 return lambda string: pattern.sub(replacement_function, string)
667
668 -def multiple_replace(string, *key_values):
669 return multiple_replacer(*key_values)(string)
670
671 # Control 672 -def check_system_error(value=1):
673 def deco_check(f): 674 def deco_f(arg, *args, **opt): 675 try: 676 return f(arg, *args, **opt) 677 except OSError, error: 678 logger.debug('try to recover from %s' % error) 679 if isinstance(arg, list): 680 prog = arg[0] 681 else: 682 prog = arg[0] 683 684 # Permission denied 685 if error.errno == 13: 686 if os.path.exists(prog): 687 os.system('chmod +x %s' % prog) 688 elif 'cwd' in opt and opt['cwd'] and \ 689 os.path.isfile(pjoin(opt['cwd'],arg[0])): 690 os.system('chmod +x %s' % pjoin(opt['cwd'],arg[0])) 691 return f(arg, *args, **opt) 692 # NO such file or directory 693 elif error.errno == 2: 694 # raise a more meaningfull error message 695 raise Exception, '%s fails with no such file or directory' \ 696 % arg 697 else: 698 raise
699 return deco_f 700 return deco_check 701
702 703 @check_system_error() 704 -def call(arg, *args, **opt):
705 """nice way to call an external program with nice error treatment""" 706 try: 707 return subprocess.call(arg, *args, **opt) 708 except OSError: 709 arg[0] = './%s' % arg[0] 710 return subprocess.call(arg, *args, **opt)
711
712 @check_system_error() 713 -def Popen(arg, *args, **opt):
714 """nice way to call an external program with nice error treatment""" 715 return subprocess.Popen(arg, *args, **opt)
716
717 @multiple_try() 718 -def mult_try_open(filepath, *args, **opt):
719 """try to open a file with multiple try to ensure that filesystem is sync""" 720 return open(filepath, *args, ** opt)
721
722 723 ################################################################################ 724 # TAIL FUNCTION 725 ################################################################################ 726 -def tail(f, n, offset=None):
727 """Reads a n lines from f with an offset of offset lines. The return 728 value is a tuple in the form ``lines``. 729 """ 730 avg_line_length = 74 731 to_read = n + (offset or 0) 732 733 while 1: 734 try: 735 f.seek(-(avg_line_length * to_read), 2) 736 except IOError: 737 # woops. apparently file is smaller than what we want 738 # to step back, go to the beginning instead 739 f.seek(0) 740 pos = f.tell() 741 lines = f.read().splitlines() 742 if len(lines) >= to_read or pos == 0: 743 return lines[-to_read:offset and -offset or None] 744 avg_line_length *= 1.3 745 avg_line_length = int(avg_line_length)
746
747 ################################################################################ 748 # LAST LINE FUNCTION 749 ################################################################################ 750 -def get_last_line(fsock):
751 """return the last line of a file""" 752 753 return tail(fsock, 1)[0]
754
755 -class BackRead(file):
756 """read a file returning the lines in reverse order for each call of readline() 757 This actually just reads blocks (4096 bytes by default) of data from the end of 758 the file and returns last line in an internal buffer.""" 759 760
761 - def readline(self):
762 """ readline in a backward way """ 763 764 while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): 765 self.blkcount = self.blkcount + 1 766 line = self.data[0] 767 try: 768 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 769 self.data = (self.read(self.blksize) + line).split('\n') 770 except IOError: # can't seek before the beginning of the file 771 self.seek(0) 772 data = self.read(self.size - (self.blksize * (self.blkcount-1))) + line 773 self.data = data.split('\n') 774 775 if len(self.data) == 0: 776 return "" 777 778 line = self.data.pop() 779 return line + '\n'
780
781 - def __init__(self, filepos, blksize=4096):
782 """initialize the internal structures""" 783 784 # get the file size 785 self.size = os.stat(filepos)[6] 786 # how big of a block to read from the file... 787 self.blksize = blksize 788 # how many blocks we've read 789 self.blkcount = 1 790 file.__init__(self, filepos, 'rb') 791 # if the file is smaller than the blocksize, read a block, 792 # otherwise, read the whole thing... 793 if self.size > self.blksize: 794 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 795 self.data = self.read(self.blksize).split('\n') 796 # strip the last item if it's empty... a byproduct of the last line having 797 # a newline at the end of it 798 if not self.data[-1]: 799 self.data.pop()
800
801 - def next(self):
802 line = self.readline() 803 if line: 804 return line 805 else: 806 raise StopIteration
807
808 809 -def write_PS_input(filePath, PS):
810 """ Write out in file filePath the PS point to be read by the MadLoop.""" 811 try: 812 PSfile = open(filePath, 'w') 813 # Add a newline in the end as the implementation fortran 'read' 814 # command on some OS is problematic if it ends directly with the 815 # floating point number read. 816 817 PSfile.write('\n'.join([' '.join(['%.16E'%pi for pi in p]) \ 818 for p in PS])+'\n') 819 PSfile.close() 820 except Exception: 821 raise MadGraph5Error, 'Could not write out the PS point to file %s.'\ 822 %str(filePath)
823
824 -def format_timer(running_time):
825 """ return a nicely string representing the time elapsed.""" 826 if running_time < 2e-2: 827 running_time = running_time = 'current time: %02dh%02d' % (time.localtime().tm_hour, time.localtime().tm_min) 828 elif running_time < 10: 829 running_time = ' %.2gs ' % running_time 830 elif 60 > running_time >= 10: 831 running_time = ' %.3gs ' % running_time 832 elif 3600 > running_time >= 60: 833 running_time = ' %im %is ' % (running_time // 60, int(running_time % 60)) 834 else: 835 running_time = ' %ih %im ' % (running_time // 3600, (running_time//60 % 60)) 836 return running_time
837
838 839 #=============================================================================== 840 # TMP_directory (designed to work as with statement) 841 #=============================================================================== 842 -class TMP_directory(object):
843 """create a temporary directory and ensure this one to be cleaned. 844 """ 845
846 - def __init__(self, suffix='', prefix='tmp', dir=None):
847 self.nb_try_remove = 0 848 import tempfile 849 self.path = tempfile.mkdtemp(suffix, prefix, dir)
850 851
852 - def __exit__(self, ctype, value, traceback ):
853 #True only for debugging: 854 if False and isinstance(value, Exception): 855 sprint("Directory %s not cleaned. This directory can be removed manually" % self.path) 856 return False 857 try: 858 shutil.rmtree(self.path) 859 except OSError: 860 self.nb_try_remove += 1 861 if self.nb_try_remove < 3: 862 time.sleep(10) 863 self.__exit__(ctype, value, traceback) 864 else: 865 logger.warning("Directory %s not completely cleaned. This directory can be removed manually" % self.path)
866
867 - def __enter__(self):
868 return self.path
869
870 -class TMP_variable(object):
871 """create a temporary directory and ensure this one to be cleaned. 872 """ 873
874 - def __init__(self, cls, attribute, value):
875 876 self.old_value = getattr(cls, attribute) 877 self.cls = cls 878 self.attribute = attribute 879 setattr(self.cls, self.attribute, value)
880
881 - def __exit__(self, ctype, value, traceback ):
882 883 setattr(self.cls, self.attribute, self.old_value)
884
885 - def __enter__(self):
886 return self.old_value
887
888 # 889 # GUNZIP/GZIP 890 # 891 -def gunzip(path, keep=False, stdout=None):
892 """ a standard replacement for os.system('gunzip -f %s.gz ' % event_path)""" 893 894 if not path.endswith(".gz"): 895 if os.path.exists("%s.gz" % path): 896 path = "%s.gz" % path 897 else: 898 raise Exception, "%(path)s does not finish by .gz and the file %(path)s.gz does not exists" %\ 899 {"path": path} 900 901 902 #for large file (>1G) it is faster and safer to use a separate thread 903 if os.path.getsize(path) > 1e8: 904 if stdout: 905 os.system('gunzip -c %s > %s' % (path, stdout)) 906 else: 907 os.system('gunzip %s' % path) 908 return 0 909 910 if not stdout: 911 stdout = path[:-3] 912 try: 913 gfile = ziplib.open(path, "r") 914 except IOError: 915 raise 916 else: 917 try: 918 open(stdout,'w').write(gfile.read()) 919 except IOError: 920 # this means that the file is actually not gzip 921 if stdout == path: 922 return 923 else: 924 files.cp(path, stdout) 925 926 if not keep: 927 os.remove(path) 928 return 0
929
930 -def gzip(path, stdout=None, error=True, forceexternal=False):
931 """ a standard replacement for os.system('gzip %s ' % path)""" 932 933 #for large file (>1G) it is faster and safer to use a separate thread 934 if os.path.getsize(path) > 1e9 or forceexternal: 935 call(['gzip', '-f', path]) 936 if stdout: 937 if not stdout.endswith(".gz"): 938 stdout = "%s.gz" % stdout 939 shutil.move('%s.gz' % path, stdout) 940 return 941 942 if not stdout: 943 stdout = "%s.gz" % path 944 elif not stdout.endswith(".gz"): 945 stdout = "%s.gz" % stdout 946 947 try: 948 ziplib.open(stdout,"w").write(open(path).read()) 949 except OverflowError: 950 gzip(path, stdout, error=error, forceexternal=True) 951 except Exception: 952 if error: 953 raise 954 else: 955 os.remove(path)
956
957 # 958 # Global function to open supported file types 959 # 960 -class open_file(object):
961 """ a convinient class to open a file """ 962 963 web_browser = None 964 eps_viewer = None 965 text_editor = None 966 configured = False 967
968 - def __init__(self, filename):
969 """open a file""" 970 971 # Check that the class is correctly configure 972 if not self.configured: 973 self.configure() 974 975 try: 976 extension = filename.rsplit('.',1)[1] 977 except IndexError: 978 extension = '' 979 980 981 # dispatch method 982 if extension in ['html','htm','php']: 983 self.open_program(self.web_browser, filename, background=True) 984 elif extension in ['ps','eps']: 985 self.open_program(self.eps_viewer, filename, background=True) 986 else: 987 self.open_program(self.text_editor,filename, mac_check=False)
988 # mac_check to False avoid to use open cmd in mac 989 990 @classmethod
991 - def configure(cls, configuration=None):
992 """ configure the way to open the file """ 993 994 cls.configured = True 995 996 # start like this is a configuration for mac 997 cls.configure_mac(configuration) 998 if sys.platform == 'darwin': 999 return # done for MAC 1000 1001 # on Mac some default (eps/web) might be kept on None. This is not 1002 #suitable for LINUX which doesn't have open command. 1003 1004 # first for eps_viewer 1005 if not cls.eps_viewer: 1006 cls.eps_viewer = cls.find_valid(['evince','gv', 'ggv'], 'eps viewer') 1007 1008 # Second for web browser 1009 if not cls.web_browser: 1010 cls.web_browser = cls.find_valid( 1011 ['firefox', 'chrome', 'safari','opera'], 1012 'web browser')
1013 1014 @classmethod
1015 - def configure_mac(cls, configuration=None):
1016 """ configure the way to open a file for mac """ 1017 1018 if configuration is None: 1019 configuration = {'text_editor': None, 1020 'eps_viewer':None, 1021 'web_browser':None} 1022 1023 for key in configuration: 1024 if key == 'text_editor': 1025 # Treat text editor ONLY text base editor !! 1026 if configuration[key]: 1027 program = configuration[key].split()[0] 1028 if not which(program): 1029 logger.warning('Specified text editor %s not valid.' % \ 1030 configuration[key]) 1031 else: 1032 # All is good 1033 cls.text_editor = configuration[key] 1034 continue 1035 #Need to find a valid default 1036 if os.environ.has_key('EDITOR'): 1037 cls.text_editor = os.environ['EDITOR'] 1038 else: 1039 cls.text_editor = cls.find_valid( 1040 ['vi', 'emacs', 'vim', 'gedit', 'nano'], 1041 'text editor') 1042 1043 elif key == 'eps_viewer': 1044 if configuration[key]: 1045 cls.eps_viewer = configuration[key] 1046 continue 1047 # else keep None. For Mac this will use the open command. 1048 elif key == 'web_browser': 1049 if configuration[key]: 1050 cls.web_browser = configuration[key] 1051 continue
1052 # else keep None. For Mac this will use the open command. 1053 1054 @staticmethod
1055 - def find_valid(possibility, program='program'):
1056 """find a valid shell program in the list""" 1057 1058 for p in possibility: 1059 if which(p): 1060 logger.info('Using default %s \"%s\". ' % (program, p) + \ 1061 'Set another one in ./input/mg5_configuration.txt') 1062 return p 1063 1064 logger.info('No valid %s found. ' % program + \ 1065 'Please set in ./input/mg5_configuration.txt') 1066 return None
1067 1068
1069 - def open_program(self, program, file_path, mac_check=True, background=False):
1070 """ open a file with a given program """ 1071 1072 if mac_check==True and sys.platform == 'darwin': 1073 return self.open_mac_program(program, file_path) 1074 1075 # Shell program only 1076 if program: 1077 arguments = program.split() # allow argument in program definition 1078 arguments.append(file_path) 1079 1080 if not background: 1081 subprocess.call(arguments) 1082 else: 1083 import thread 1084 thread.start_new_thread(subprocess.call,(arguments,)) 1085 else: 1086 logger.warning('Not able to open file %s since no program configured.' % file_path + \ 1087 'Please set one in ./input/mg5_configuration.txt')
1088
1089 - def open_mac_program(self, program, file_path):
1090 """ open a text with the text editor """ 1091 1092 if not program: 1093 # Ask to mac manager 1094 os.system('open %s' % file_path) 1095 elif which(program): 1096 # shell program 1097 arguments = program.split() # Allow argument in program definition 1098 arguments.append(file_path) 1099 subprocess.call(arguments) 1100 else: 1101 # not shell program 1102 os.system('open -a %s %s' % (program, file_path))
1103
1104 -def is_executable(path):
1105 """ check if a path is executable""" 1106 try: 1107 return os.access(path, os.X_OK) 1108 except Exception: 1109 return False
1110
1111 -class OptionParser(optparse.OptionParser):
1112 """Option Peaser which raise an error instead as calling exit""" 1113
1114 - def exit(self, status=0, msg=None):
1115 if msg: 1116 raise InvalidCmd, msg 1117 else: 1118 raise InvalidCmd
1119
1120 -def sprint(*args, **opt):
1121 """Returns the current line number in our program.""" 1122 1123 if not __debug__: 1124 return 1125 1126 import inspect 1127 if opt.has_key('log'): 1128 log = opt['log'] 1129 else: 1130 log = logging.getLogger('madgraph') 1131 if opt.has_key('level'): 1132 level = opt['level'] 1133 else: 1134 level = logging.getLogger('madgraph').level 1135 #print "madgraph level",level 1136 #if level == 20: 1137 # level = 10 #avoid info level 1138 #print "use", level 1139 lineno = inspect.currentframe().f_back.f_lineno 1140 fargs = inspect.getframeinfo(inspect.currentframe().f_back) 1141 filename, lineno = fargs[:2] 1142 #file = inspect.currentframe().f_back.co_filename 1143 #print type(file) 1144 try: 1145 source = inspect.getsourcelines(inspect.currentframe().f_back) 1146 line = source[0][lineno-source[1]] 1147 line = re.findall(r"misc\.sprint\(\s*(.*)\)\s*($|#)", line)[0][0] 1148 if line.startswith("'") and line.endswith("'") and line.count(",") ==0: 1149 line= '' 1150 elif line.startswith("\"") and line.endswith("\"") and line.count(",") ==0: 1151 line= '' 1152 elif line.startswith(("\"","'")) and len(args)==1 and "%" in line: 1153 line= '' 1154 except Exception: 1155 line='' 1156 1157 if line: 1158 intro = ' %s = \033[0m' % line 1159 else: 1160 intro = '' 1161 1162 1163 log.log(level, ' '.join([intro]+[str(a) for a in args]) + \ 1164 ' \033[1;30m[%s at line %s]\033[0m' % (os.path.basename(filename), lineno)) 1165 return
1166
1167 ################################################################################ 1168 # function to check if two float are approximatively equal 1169 ################################################################################ 1170 -def equal(a,b,sig_fig=6, zero_limit=True):
1171 """function to check if two float are approximatively equal""" 1172 import math 1173 1174 if not a or not b: 1175 if zero_limit: 1176 power = sig_fig + 1 1177 else: 1178 return a == b 1179 else: 1180 power = sig_fig - int(math.log10(abs(a))) + 1 1181 1182 return ( a==b or abs(int(a*10**power) - int(b*10**power)) < 10)
1183
1184 ################################################################################ 1185 # class to change directory with the "with statement" 1186 ################################################################################ 1187 -class chdir:
1188 - def __init__(self, newPath):
1189 self.newPath = newPath
1190
1191 - def __enter__(self):
1192 self.savedPath = os.getcwd() 1193 os.chdir(self.newPath)
1194
1195 - def __exit__(self, etype, value, traceback):
1196 os.chdir(self.savedPath)
1197
1198 ################################################################################ 1199 # Timeout FUNCTION 1200 ################################################################################ 1201 1202 -def timeout(func, args=(), kwargs={}, timeout_duration=1, default=None):
1203 '''This function will spwan a thread and run the given function using the args, kwargs and 1204 return the given default value if the timeout_duration is exceeded 1205 ''' 1206 import threading 1207 class InterruptableThread(threading.Thread): 1208 def __init__(self): 1209 threading.Thread.__init__(self) 1210 self.result = default
1211 def run(self): 1212 try: 1213 self.result = func(*args, **kwargs) 1214 except Exception,error: 1215 print error 1216 self.result = default 1217 it = InterruptableThread() 1218 it.start() 1219 it.join(timeout_duration) 1220 return it.result 1221
1222 1223 ################################################################################ 1224 # TAIL FUNCTION 1225 ################################################################################ 1226 -class digest:
1227
1228 - def test_all(self):
1229 try: 1230 return self.test_hashlib() 1231 except Exception: 1232 pass 1233 try: 1234 return self.test_md5() 1235 except Exception: 1236 pass 1237 try: 1238 return self.test_zlib() 1239 except Exception: 1240 pass
1241
1242 - def test_hashlib(self):
1243 import hashlib 1244 def digest(text): 1245 """using mg5 for the hash""" 1246 t = hashlib.md5() 1247 t.update(text) 1248 return t.hexdigest()
1249 return digest
1250
1251 - def test_md5(self):
1252 import md5 1253 def digest(text): 1254 """using mg5 for the hash""" 1255 t = md5.md5() 1256 t.update(text) 1257 return t.hexdigest()
1258 return digest 1259
1260 - def test_zlib(self):
1261 import zlib 1262 def digest(text): 1263 return zlib.adler32(text)
1264 1265 digest = digest().test_all()
1266 1267 #=============================================================================== 1268 # Helper class for timing and RAM flashing of subprocesses. 1269 #=============================================================================== 1270 -class ProcessTimer:
1271 - def __init__(self,*args,**opts):
1272 self.cmd_args = args 1273 self.cmd_opts = opts 1274 self.execution_state = False
1275
1276 - def execute(self):
1277 self.max_vms_memory = 0 1278 self.max_rss_memory = 0 1279 1280 self.t1 = None 1281 self.t0 = time.time() 1282 self.p = subprocess.Popen(*self.cmd_args,**self.cmd_opts) 1283 self.execution_state = True
1284
1285 - def poll(self):
1286 if not self.check_execution_state(): 1287 return False 1288 1289 self.t1 = time.time() 1290 flash = subprocess.Popen("ps -p %i -o rss"%self.p.pid, 1291 shell=True,stdout=subprocess.PIPE) 1292 stdout_list = flash.communicate()[0].split('\n') 1293 rss_memory = int(stdout_list[1]) 1294 # for now we ignore vms 1295 vms_memory = 0 1296 1297 # This is the neat version using psutil 1298 # try: 1299 # pp = psutil.Process(self.p.pid) 1300 # 1301 # # obtain a list of the subprocess and all its descendants 1302 # descendants = list(pp.get_children(recursive=True)) 1303 # descendants = descendants + [pp] 1304 # 1305 # rss_memory = 0 1306 # vms_memory = 0 1307 # 1308 # # calculate and sum up the memory of the subprocess and all its descendants 1309 # for descendant in descendants: 1310 # try: 1311 # mem_info = descendant.get_memory_info() 1312 # 1313 # rss_memory += mem_info[0] 1314 # vms_memory += mem_info[1] 1315 # except psutil.error.NoSuchProcess: 1316 # # sometimes a subprocess descendant will have terminated between the time 1317 # # we obtain a list of descendants, and the time we actually poll this 1318 # # descendant's memory usage. 1319 # pass 1320 # 1321 # except psutil.error.NoSuchProcess: 1322 # return self.check_execution_state() 1323 1324 self.max_vms_memory = max(self.max_vms_memory,vms_memory) 1325 self.max_rss_memory = max(self.max_rss_memory,rss_memory) 1326 1327 return self.check_execution_state()
1328
1329 - def is_running(self):
1330 # Version with psutil 1331 # return psutil.pid_exists(self.p.pid) and self.p.poll() == None 1332 return self.p.poll() == None
1333
1334 - def check_execution_state(self):
1335 if not self.execution_state: 1336 return False 1337 if self.is_running(): 1338 return True 1339 self.executation_state = False 1340 self.t1 = time.time() 1341 return False
1342
1343 - def close(self,kill=False):
1344 1345 if self.p.poll() == None: 1346 if kill: 1347 self.p.kill() 1348 else: 1349 self.p.terminate()
1350 1351 # Again a neater handling with psutil 1352 # try: 1353 # pp = psutil.Process(self.p.pid) 1354 # if kill: 1355 # pp.kill() 1356 # else: 1357 # pp.terminate() 1358 # except psutil.error.NoSuchProcess: 1359 # pass 1360 1361 1362 try: 1363 import Foundation 1364 import objc 1365 NSUserNotification = objc.lookUpClass('NSUserNotification') 1366 NSUserNotificationCenter = objc.lookUpClass('NSUserNotificationCenter')
1367 1368 - def apple_notify(subtitle, info_text, userInfo={}):
1369 try: 1370 notification = NSUserNotification.alloc().init() 1371 notification.setTitle_('MadGraph5_aMC@NLO') 1372 notification.setSubtitle_(subtitle) 1373 notification.setInformativeText_(info_text) 1374 notification.setUserInfo_(userInfo) 1375 NSUserNotificationCenter.defaultUserNotificationCenter().scheduleNotification_(notification) 1376 except: 1377 pass
1378 except:
1379 - def apple_notify(subtitle, info_text, userInfo={}):
1380 return
1381