Package madgraph :: Package various :: Module misc
[hide private]
[frames] | no frames]

Source Code for Module madgraph.various.misc

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2009 The MadGraph5_aMC@NLO Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph5_aMC@NLO project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph5_aMC@NLO license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, visit madgraph.phys.ucl.ac.be and amcatnlo.web.cern.ch 
  13  # 
  14  ################################################################################ 
  15   
  16  """A set of functions performing routine administrative I/O tasks.""" 
  17   
  18  import logging 
  19  import os 
  20  import re 
  21  import signal 
  22  import subprocess 
  23  import sys 
  24  import StringIO 
  25  import sys 
  26  import optparse 
  27  import time 
  28  import shutil 
  29  import traceback 
  30  import gzip as ziplib 
  31   
  32  try: 
  33      # Use in MadGraph 
  34      import madgraph 
  35      from madgraph import MadGraph5Error, InvalidCmd 
  36      import madgraph.iolibs.files as files 
  37  except Exception, error: 
  38      if __debug__: 
  39          print error 
  40      # Use in MadEvent 
  41      import internal as madgraph 
  42      from internal import MadGraph5Error, InvalidCmd 
  43      import internal.files as files 
  44       
  45  logger = logging.getLogger('cmdprint.ext_program') 
  46  logger_stderr = logging.getLogger('madevent.misc') 
  47  pjoin = os.path.join 
48 49 #=============================================================================== 50 # parse_info_str 51 #=============================================================================== 52 -def parse_info_str(fsock):
53 """Parse a newline separated list of "param=value" as a dictionnary 54 """ 55 56 info_dict = {} 57 pattern = re.compile("(?P<name>\w*)\s*=\s*(?P<value>.*)", 58 re.IGNORECASE | re.VERBOSE) 59 for entry in fsock: 60 entry = entry.strip() 61 if len(entry) == 0: continue 62 m = pattern.match(entry) 63 if m is not None: 64 info_dict[m.group('name')] = m.group('value') 65 else: 66 raise IOError, "String %s is not a valid info string" % entry 67 68 return info_dict
69
70 71 #=============================================================================== 72 # mute_logger (designed to be a decorator) 73 #=============================================================================== 74 -def mute_logger(names=['madgraph','ALOHA','cmdprint','madevent'], levels=[50,50,50,50]):
75 """change the logger level and restore those at their initial value at the 76 end of the function decorated.""" 77 def control_logger(f): 78 def restore_old_levels(names, levels): 79 for name, level in zip(names, levels): 80 log_module = logging.getLogger(name) 81 log_module.setLevel(level)
82 83 def f_with_no_logger(self, *args, **opt): 84 old_levels = [] 85 for name, level in zip(names, levels): 86 log_module = logging.getLogger(name) 87 old_levels.append(log_module.level) 88 log_module.setLevel(level) 89 try: 90 out = f(self, *args, **opt) 91 restore_old_levels(names, old_levels) 92 return out 93 except: 94 restore_old_levels(names, old_levels) 95 raise 96 97 return f_with_no_logger 98 return control_logger 99
100 #=============================================================================== 101 # get_pkg_info 102 #=============================================================================== 103 -def get_pkg_info(info_str=None):
104 """Returns the current version information of the MadGraph5_aMC@NLO package, 105 as written in the VERSION text file. If the file cannot be found, 106 a dictionary with empty values is returned. As an option, an info 107 string can be passed to be read instead of the file content. 108 """ 109 110 if info_str is None: 111 info_dict = files.read_from_file(os.path.join(madgraph.__path__[0], 112 "VERSION"), 113 parse_info_str, 114 print_error=False) 115 else: 116 info_dict = parse_info_str(StringIO.StringIO(info_str)) 117 118 return info_dict
119
120 #=============================================================================== 121 # get_time_info 122 #=============================================================================== 123 -def get_time_info():
124 """Returns the present time info for use in MG5 command history header. 125 """ 126 127 creation_time = time.asctime() 128 time_info = {'time': creation_time, 129 'fill': ' ' * (26 - len(creation_time))} 130 131 return time_info
132
133 #=============================================================================== 134 # Find the subdirectory which includes the files ending with a given extension 135 #=============================================================================== 136 -def find_includes_path(start_path, extension):
137 """Browse the subdirectories of the path 'start_path' and returns the first 138 one found which contains at least one file ending with the string extension 139 given in argument.""" 140 141 subdirs=[pjoin(start_path,dir) for dir in os.listdir(start_path)] 142 for subdir in subdirs: 143 if os.path.isfile(subdir): 144 if os.path.basename(subdir).endswith(extension): 145 return start_path 146 elif os.path.isdir(subdir): 147 return find_includes_path(subdir, extension) 148 return None
149
150 #=============================================================================== 151 # find a executable 152 #=============================================================================== 153 -def which(program):
154 def is_exe(fpath): 155 return os.path.exists(fpath) and os.access(\ 156 os.path.realpath(fpath), os.X_OK)
157 158 if not program: 159 return None 160 161 fpath, fname = os.path.split(program) 162 if fpath: 163 if is_exe(program): 164 return program 165 else: 166 for path in os.environ["PATH"].split(os.pathsep): 167 exe_file = os.path.join(path, program) 168 if is_exe(exe_file): 169 return exe_file 170 return None 171
172 #=============================================================================== 173 # find a library 174 #=============================================================================== 175 -def which_lib(lib):
176 def is_lib(fpath): 177 return os.path.exists(fpath) and os.access(fpath, os.R_OK)
178 179 if not lib: 180 return None 181 182 fpath, fname = os.path.split(lib) 183 if fpath: 184 if is_lib(lib): 185 return lib 186 else: 187 locations = sum([os.environ[env_path].split(os.pathsep) for env_path in 188 ["DYLD_LIBRARY_PATH","LD_LIBRARY_PATH","LIBRARY_PATH","PATH"] 189 if env_path in os.environ],[]) 190 for path in locations: 191 lib_file = os.path.join(path, lib) 192 if is_lib(lib_file): 193 return lib_file 194 return None 195
196 #=============================================================================== 197 # Return Nice display for a random variable 198 #=============================================================================== 199 -def nice_representation(var, nb_space=0):
200 """ Return nice information on the current variable """ 201 202 #check which data to put: 203 info = [('type',type(var)),('str', var)] 204 if hasattr(var, 'func_doc'): 205 info.append( ('DOC', var.func_doc) ) 206 if hasattr(var, '__doc__'): 207 info.append( ('DOC', var.__doc__) ) 208 if hasattr(var, '__dict__'): 209 info.append( ('ATTRIBUTE', var.__dict__.keys() )) 210 211 spaces = ' ' * nb_space 212 213 outstr='' 214 for name, value in info: 215 outstr += '%s%3s : %s\n' % (spaces,name, value) 216 217 return outstr
218 219 # 220 # Decorator for re-running a crashing function automatically. 221 # 222 wait_once = False
223 -def multiple_try(nb_try=5, sleep=20):
224 225 def deco_retry(f): 226 def deco_f_retry(*args, **opt): 227 for i in range(nb_try): 228 try: 229 return f(*args, **opt) 230 except KeyboardInterrupt: 231 raise 232 except Exception, error: 233 global wait_once 234 if not wait_once: 235 text = """Start waiting for update. (more info in debug mode)""" 236 logger.info(text) 237 logger_stderr.debug('fail to do %s function with %s args. %s try on a max of %s (%s waiting time)' % 238 (str(f), ', '.join([str(a) for a in args]), i+1, nb_try, sleep * (i+1))) 239 logger_stderr.debug('error is %s' % str(error)) 240 if __debug__: logger_stderr.debug('and occurred at :'+traceback.format_exc()) 241 wait_once = True 242 time.sleep(sleep * (i+1)) 243 244 if __debug__: 245 raise 246 raise error.__class__, '[Fail %i times] \n %s ' % (i+1, error)
247 return deco_f_retry 248 return deco_retry 249
250 #=============================================================================== 251 # Compiler which returns smart output error in case of trouble 252 #=============================================================================== 253 -def compile(arg=[], cwd=None, mode='fortran', job_specs = True, nb_core=1 ,**opt):
254 """compile a given directory""" 255 256 cmd = ['make'] 257 try: 258 if nb_core > 1: 259 cmd.append('-j%s' % nb_core) 260 cmd += arg 261 p = subprocess.Popen(cmd, stdout=subprocess.PIPE, 262 stderr=subprocess.STDOUT, cwd=cwd, **opt) 263 (out, err) = p.communicate() 264 except OSError, error: 265 if cwd and not os.path.exists(cwd): 266 raise OSError, 'Directory %s doesn\'t exists. Impossible to run make' % cwd 267 else: 268 error_text = "Impossible to compile %s directory\n" % cwd 269 error_text += "Trying to launch make command returns:\n" 270 error_text += " " + str(error) + "\n" 271 error_text += "In general this means that your computer is not able to compile." 272 if sys.platform == "darwin": 273 error_text += "Note that MacOSX doesn\'t have gmake/gfortan install by default.\n" 274 error_text += "Xcode3 contains those required programs" 275 raise MadGraph5Error, error_text 276 277 if p.returncode: 278 # Check that makefile exists 279 if not cwd: 280 cwd = os.getcwd() 281 all_file = [f.lower() for f in os.listdir(cwd)] 282 if 'makefile' not in all_file: 283 raise OSError, 'no makefile present in %s' % os.path.realpath(cwd) 284 285 if mode == 'fortran' and not (which('g77') or which('gfortran')): 286 error_msg = 'A fortran compiler (g77 or gfortran) is required to create this output.\n' 287 error_msg += 'Please install g77 or gfortran on your computer and retry.' 288 raise MadGraph5Error, error_msg 289 elif mode == 'cpp' and not which('g++'): 290 error_msg ='A C++ compiler (g++) is required to create this output.\n' 291 error_msg += 'Please install g++ (which is part of the gcc package) on your computer and retry.' 292 raise MadGraph5Error, error_msg 293 294 # Check if this is due to the need of gfortran 4.6 for quadruple precision 295 if any(tag.upper() in out.upper() for tag in ['real(kind=16)','real*16', 296 'complex*32']) and mode == 'fortran' and not \ 297 ''.join(get_gfortran_version().split('.')) >= '46': 298 if not which('gfortran'): 299 raise MadGraph5Error, 'The fortran compiler gfortran v4.6 or later '+\ 300 'is required to compile %s.\nPlease install it and retry.'%cwd 301 else: 302 logger_stderr.error('ERROR, you could not compile %s because'%cwd+\ 303 ' your version of gfortran is older than 4.6. MadGraph5_aMC@NLO will carry on,'+\ 304 ' but will not be able to compile an executable.') 305 return p.returncode 306 # Other reason 307 error_text = 'A compilation Error occurs ' 308 if cwd: 309 error_text += 'when trying to compile %s.\n' % cwd 310 error_text += 'The compilation fails with the following output message:\n' 311 error_text += ' '+out.replace('\n','\n ')+'\n' 312 error_text += 'Please try to fix this compilations issue and retry.\n' 313 error_text += 'Help might be found at https://answers.launchpad.net/madgraph5.\n' 314 error_text += 'If you think that this is a bug, you can report this at https://bugs.launchpad.net/madgraph5' 315 raise MadGraph5Error, error_text 316 return p.returncode
317
318 -def get_gfortran_version(compiler='gfortran'):
319 """ Returns the gfortran version as a string. 320 Returns '0' if it failed.""" 321 try: 322 p = Popen([compiler, '-dumpversion'], stdout=subprocess.PIPE, 323 stderr=subprocess.PIPE) 324 output, error = p.communicate() 325 version_finder=re.compile(r"(?P<version>(\d.)*\d)") 326 version = version_finder.search(output).group('version') 327 return version 328 except Exception: 329 return '0'
330
331 -def mod_compilator(directory, new='gfortran', current=None, compiler_type='gfortran'):
332 #define global regular expression 333 if type(directory)!=list: 334 directory=[directory] 335 336 #search file 337 file_to_change=find_makefile_in_dir(directory) 338 if compiler_type == 'gfortran': 339 comp_re = re.compile('^(\s*)FC\s*=\s*(.+)\s*$') 340 var = 'FC' 341 elif compiler_type == 'cpp': 342 comp_re = re.compile('^(\s*)CXX\s*=\s*(.+)\s*$') 343 var = 'CXX' 344 else: 345 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 346 347 mod = False 348 for name in file_to_change: 349 lines = open(name,'r').read().split('\n') 350 for iline, line in enumerate(lines): 351 result = comp_re.match(line) 352 if result: 353 if new != result.group(2): 354 mod = True 355 lines[iline] = result.group(1) + var + "=" + new 356 if mod: 357 open(name,'w').write('\n'.join(lines)) 358 # reset it to change the next file 359 mod = False
360
361 #=============================================================================== 362 # mute_logger (designed to work as with statement) 363 #=============================================================================== 364 -class MuteLogger(object):
365 """mute_logger (designed to work as with statement), 366 files allow to redirect the output of the log to a given file. 367 """ 368
369 - def __init__(self, names, levels, files=None, **opt):
370 assert isinstance(names, list) 371 assert isinstance(names, list) 372 373 self.names = names 374 self.levels = levels 375 if isinstance(files, list): 376 self.files = files 377 else: 378 self.files = [files] * len(names) 379 self.logger_saved_info = {} 380 self.opts = opt
381
382 - def __enter__(self):
383 old_levels = [] 384 for name, level, path in zip(self.names, self.levels, self.files): 385 if path: 386 self.setup_logFile_for_logger(path, name, **self.opts) 387 log_module = logging.getLogger(name) 388 old_levels.append(log_module.level) 389 log_module = logging.getLogger(name) 390 log_module.setLevel(level) 391 self.levels = old_levels
392
393 - def __exit__(self, ctype, value, traceback ):
394 for name, level, path, level in zip(self.names, self.levels, self.files, self.levels): 395 if 'keep' in self.opts and not self.opts['keep']: 396 self.restore_logFile_for_logger(name, level, path=path) 397 else: 398 self.restore_logFile_for_logger(name, level) 399 400 log_module = logging.getLogger(name) 401 log_module.setLevel(level)
402
403 - def setup_logFile_for_logger(self, path, full_logname, **opts):
404 """ Setup the logger by redirecting them all to logfiles in tmp """ 405 406 logs = full_logname.split('.') 407 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 408 range(len(full_logname.split('.')))] 409 for logname in lognames: 410 try: 411 os.remove(path) 412 except Exception, error: 413 pass 414 my_logger = logging.getLogger(logname) 415 hdlr = logging.FileHandler(path) 416 # I assume below that the orders of the handlers in my_logger.handlers 417 # remains the same after having added/removed the FileHandler 418 self.logger_saved_info[logname] = [hdlr, my_logger.handlers] 419 #for h in my_logger.handlers: 420 # h.setLevel(logging.CRITICAL) 421 for old_hdlr in list(my_logger.handlers): 422 my_logger.removeHandler(old_hdlr) 423 my_logger.addHandler(hdlr) 424 #my_logger.setLevel(level) 425 my_logger.debug('Log of %s' % logname)
426
427 - def restore_logFile_for_logger(self, full_logname, level, path=None, **opts):
428 """ Setup the logger by redirecting them all to logfiles in tmp """ 429 430 logs = full_logname.split('.') 431 lognames = [ '.'.join(logs[:(len(logs)-i)]) for i in\ 432 range(len(full_logname.split('.')))] 433 for logname in lognames: 434 if path: 435 try: 436 os.remove(path) 437 except Exception, error: 438 pass 439 my_logger = logging.getLogger(logname) 440 if logname in self.logger_saved_info: 441 my_logger.removeHandler(self.logger_saved_info[logname][0]) 442 for old_hdlr in self.logger_saved_info[logname][1]: 443 my_logger.addHandler(old_hdlr) 444 else: 445 my_logger.setLevel(level)
446
447 #for i, h in enumerate(my_logger.handlers): 448 # h.setLevel(cls.logger_saved_info[logname][2][i]) 449 450 451 -def detect_current_compiler(path, compiler_type='fortran'):
452 """find the current compiler for the current directory""" 453 454 # comp = re.compile("^\s*FC\s*=\s*(\w+)\s*") 455 # The regular expression below allows for compiler definition with absolute path 456 if compiler_type == 'fortran': 457 comp = re.compile("^\s*FC\s*=\s*([\w\/\\.\-]+)\s*") 458 elif compiler_type == 'cpp': 459 comp = re.compile("^\s*CXX\s*=\s*([\w\/\\.\-]+)\s*") 460 else: 461 MadGraph5Error, 'Unknown compiler type: %s' % compiler_type 462 463 for line in open(path): 464 if comp.search(line): 465 compiler = comp.search(line).groups()[0] 466 return compiler
467
468 -def find_makefile_in_dir(directory):
469 """ return a list of all file starting with makefile in the given directory""" 470 471 out=[] 472 #list mode 473 if type(directory)==list: 474 for name in directory: 475 out+=find_makefile_in_dir(name) 476 return out 477 478 #single mode 479 for name in os.listdir(directory): 480 if os.path.isdir(directory+'/'+name): 481 out+=find_makefile_in_dir(directory+'/'+name) 482 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('makefile'): 483 out.append(directory+'/'+name) 484 elif os.path.isfile(directory+'/'+name) and name.lower().startswith('make_opt'): 485 out.append(directory+'/'+name) 486 return out
487
488 -def rm_old_compile_file():
489 490 # remove all the .o files 491 os.path.walk('.', rm_file_extension, '.o') 492 493 # remove related libraries 494 libraries = ['libblocks.a', 'libgeneric_mw.a', 'libMWPS.a', 'libtools.a', 'libdhelas3.a', 495 'libdsample.a', 'libgeneric.a', 'libmodel.a', 'libpdf.a', 'libdhelas3.so', 'libTF.a', 496 'libdsample.so', 'libgeneric.so', 'libmodel.so', 'libpdf.so'] 497 lib_pos='./lib' 498 [os.remove(os.path.join(lib_pos, lib)) for lib in libraries \ 499 if os.path.exists(os.path.join(lib_pos, lib))]
500
501 502 -def format_time(n_secs):
503 m, s = divmod(n_secs, 60) 504 h, m = divmod(m, 60) 505 d, h = divmod(h, 24) 506 if d > 0: 507 return "%d day%s,%dh%02dm%02ds" % (d,'' if d<=1 else 's',h, m, s) 508 elif h > 0: 509 return "%dh%02dm%02ds" % (h, m, s) 510 elif m > 0: 511 return "%dm%02ds" % (m, s) 512 else: 513 return "%d second%s" % (s, '' if s<=1 else 's')
514
515 -def rm_file_extension( ext, dirname, names):
516 517 [os.remove(os.path.join(dirname, name)) for name in names if name.endswith(ext)]
518
519 520 521 -def multiple_replacer(*key_values):
522 replace_dict = dict(key_values) 523 replacement_function = lambda match: replace_dict[match.group(0)] 524 pattern = re.compile("|".join([re.escape(k) for k, v in key_values]), re.M) 525 return lambda string: pattern.sub(replacement_function, string)
526
527 -def multiple_replace(string, *key_values):
528 return multiple_replacer(*key_values)(string)
529
530 # Control 531 -def check_system_error(value=1):
532 def deco_check(f): 533 def deco_f(arg, *args, **opt): 534 try: 535 return f(arg, *args, **opt) 536 except OSError, error: 537 logger.debug('try to recover from %s' % error) 538 if isinstance(arg, list): 539 prog = arg[0] 540 else: 541 prog = arg[0] 542 543 # Permission denied 544 if error.errno == 13: 545 if os.path.exists(prog): 546 os.system('chmod +x %s' % prog) 547 elif 'cwd' in opt and opt['cwd'] and \ 548 os.path.isfile(pjoin(opt['cwd'],arg[0])): 549 os.system('chmod +x %s' % pjoin(opt['cwd'],arg[0])) 550 return f(arg, *args, **opt) 551 # NO such file or directory 552 elif error.errno == 2: 553 # raise a more meaningfull error message 554 raise Exception, '%s fails with no such file or directory' \ 555 % arg 556 else: 557 raise
558 return deco_f 559 return deco_check 560
561 562 @check_system_error() 563 -def call(arg, *args, **opt):
564 """nice way to call an external program with nice error treatment""" 565 try: 566 return subprocess.call(arg, *args, **opt) 567 except OSError: 568 arg[0] = './%s' % arg[0] 569 return subprocess.call(arg, *args, **opt)
570
571 @check_system_error() 572 -def Popen(arg, *args, **opt):
573 """nice way to call an external program with nice error treatment""" 574 return subprocess.Popen(arg, *args, **opt)
575
576 @multiple_try() 577 -def mult_try_open(filepath, *args, **opt):
578 """try to open a file with multiple try to ensure that filesystem is sync""" 579 return open(filepath, *args, ** opt)
580
581 582 ################################################################################ 583 # TAIL FUNCTION 584 ################################################################################ 585 -def tail(f, n, offset=None):
586 """Reads a n lines from f with an offset of offset lines. The return 587 value is a tuple in the form ``lines``. 588 """ 589 avg_line_length = 74 590 to_read = n + (offset or 0) 591 592 while 1: 593 try: 594 f.seek(-(avg_line_length * to_read), 2) 595 except IOError: 596 # woops. apparently file is smaller than what we want 597 # to step back, go to the beginning instead 598 f.seek(0) 599 pos = f.tell() 600 lines = f.read().splitlines() 601 if len(lines) >= to_read or pos == 0: 602 return lines[-to_read:offset and -offset or None] 603 avg_line_length *= 1.3 604 avg_line_length = int(avg_line_length)
605
606 ################################################################################ 607 # LAST LINE FUNCTION 608 ################################################################################ 609 -def get_last_line(fsock):
610 """return the last line of a file""" 611 612 return tail(fsock, 1)[0]
613
614 -class BackRead(file):
615 """read a file returning the lines in reverse order for each call of readline() 616 This actually just reads blocks (4096 bytes by default) of data from the end of 617 the file and returns last line in an internal buffer.""" 618 619
620 - def readline(self):
621 """ readline in a backward way """ 622 623 while len(self.data) == 1 and ((self.blkcount * self.blksize) < self.size): 624 self.blkcount = self.blkcount + 1 625 line = self.data[0] 626 try: 627 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 628 self.data = (self.read(self.blksize) + line).split('\n') 629 except IOError: # can't seek before the beginning of the file 630 self.seek(0) 631 data = self.read(self.size - (self.blksize * (self.blkcount-1))) + line 632 self.data = data.split('\n') 633 634 if len(self.data) == 0: 635 return "" 636 637 line = self.data.pop() 638 return line + '\n'
639
640 - def __init__(self, filepos, blksize=4096):
641 """initialize the internal structures""" 642 643 # get the file size 644 self.size = os.stat(filepos)[6] 645 # how big of a block to read from the file... 646 self.blksize = blksize 647 # how many blocks we've read 648 self.blkcount = 1 649 file.__init__(self, filepos, 'rb') 650 # if the file is smaller than the blocksize, read a block, 651 # otherwise, read the whole thing... 652 if self.size > self.blksize: 653 self.seek(-self.blksize * self.blkcount, 2) # read from end of file 654 self.data = self.read(self.blksize).split('\n') 655 # strip the last item if it's empty... a byproduct of the last line having 656 # a newline at the end of it 657 if not self.data[-1]: 658 self.data.pop()
659
660 - def next(self):
661 line = self.readline() 662 if line: 663 return line 664 else: 665 raise StopIteration
666
667 668 -def write_PS_input(filePath, PS):
669 """ Write out in file filePath the PS point to be read by the MadLoop.""" 670 try: 671 PSfile = open(filePath, 'w') 672 # Add a newline in the end as the implementation fortran 'read' 673 # command on some OS is problematic if it ends directly with the 674 # floating point number read. 675 676 PSfile.write('\n'.join([' '.join(['%.16E'%pi for pi in p]) \ 677 for p in PS])+'\n') 678 PSfile.close() 679 except Exception: 680 raise MadGraph5Error, 'Could not write out the PS point to file %s.'\ 681 %str(filePath)
682
683 -def format_timer(running_time):
684 """ return a nicely string representing the time elapsed.""" 685 if running_time < 2e-2: 686 running_time = running_time = 'current time: %02dh%02d' % (time.localtime().tm_hour, time.localtime().tm_min) 687 elif running_time < 10: 688 running_time = ' %.2gs ' % running_time 689 elif 60 > running_time >= 10: 690 running_time = ' %.3gs ' % running_time 691 elif 3600 > running_time >= 60: 692 running_time = ' %im %is ' % (running_time // 60, int(running_time % 60)) 693 else: 694 running_time = ' %ih %im ' % (running_time // 3600, (running_time//60 % 60)) 695 return running_time
696
697 698 #=============================================================================== 699 # TMP_directory (designed to work as with statement) 700 #=============================================================================== 701 -class TMP_directory(object):
702 """create a temporary directory and ensure this one to be cleaned. 703 """ 704
705 - def __init__(self, suffix='', prefix='tmp', dir=None):
706 self.nb_try_remove = 0 707 import tempfile 708 self.path = tempfile.mkdtemp(suffix, prefix, dir)
709 710
711 - def __exit__(self, ctype, value, traceback ):
712 try: 713 shutil.rmtree(self.path) 714 except OSError: 715 self.nb_try_remove += 1 716 if self.nb_try_remove < 3: 717 time.sleep(10) 718 self.__exit__(ctype, value, traceback) 719 else: 720 logger.warning("Directory %s not completely cleaned. This directory can be removed manually" % self.path)
721
722 - def __enter__(self):
723 return self.path
724 #
725 # GUNZIP/GZIP 726 # 727 -def gunzip(path, keep=False, stdout=None):
728 """ a standard replacement for os.system('gunzip -f %s.gz ' % event_path)""" 729 730 if not path.endswith(".gz"): 731 if os.path.exists("%s.gz" % path): 732 path = "%s.gz" % path 733 else: 734 raise Exception, "%(path)s does not finish by .gz and the file %(path)s.gz does not exists" %\ 735 {"path": path} 736 737 738 #for large file (>1G) it is faster and safer to use a separate thread 739 if os.path.getsize(path) > 1e8: 740 if stdout: 741 os.system('gunzip -c %s > %s' % (path, stdout)) 742 else: 743 os.system('gunzip %s' % path) 744 return 0 745 746 if not stdout: 747 stdout = path[:-3] 748 try: 749 gfile = ziplib.open(path, "r") 750 except IOError: 751 raise 752 else: 753 try: 754 open(stdout,'w').write(gfile.read()) 755 except IOError: 756 # this means that the file is actually not gzip 757 if stdout == path: 758 return 759 else: 760 files.cp(path, stdout) 761 762 if not keep: 763 os.remove(path) 764 return 0
765
766 -def gzip(path, stdout=None, error=True, forceexternal=False):
767 """ a standard replacement for os.system('gzip %s ' % path)""" 768 769 #for large file (>1G) it is faster and safer to use a separate thread 770 if os.path.getsize(path) > 1e9 or forceexternal: 771 call(['gzip', '-f', path]) 772 if stdout: 773 if not stdout.endswith(".gz"): 774 stdout = "%s.gz" % stdout 775 shutil.move('%s.gz' % path, stdout) 776 return 777 778 if not stdout: 779 stdout = "%s.gz" % path 780 elif not stdout.endswith(".gz"): 781 stdout = "%s.gz" % stdout 782 783 try: 784 ziplib.open(stdout,"w").write(open(path).read()) 785 except OverflowError: 786 gzip(path, stdout, error=error, forceexternal=True) 787 except Exception: 788 if error: 789 raise 790 else: 791 os.remove(path)
792
793 # 794 # Global function to open supported file types 795 # 796 -class open_file(object):
797 """ a convinient class to open a file """ 798 799 web_browser = None 800 eps_viewer = None 801 text_editor = None 802 configured = False 803
804 - def __init__(self, filename):
805 """open a file""" 806 807 # Check that the class is correctly configure 808 if not self.configured: 809 self.configure() 810 811 try: 812 extension = filename.rsplit('.',1)[1] 813 except IndexError: 814 extension = '' 815 816 817 # dispatch method 818 if extension in ['html','htm','php']: 819 self.open_program(self.web_browser, filename, background=True) 820 elif extension in ['ps','eps']: 821 self.open_program(self.eps_viewer, filename, background=True) 822 else: 823 self.open_program(self.text_editor,filename, mac_check=False)
824 # mac_check to False avoid to use open cmd in mac 825 826 @classmethod
827 - def configure(cls, configuration=None):
828 """ configure the way to open the file """ 829 830 cls.configured = True 831 832 # start like this is a configuration for mac 833 cls.configure_mac(configuration) 834 if sys.platform == 'darwin': 835 return # done for MAC 836 837 # on Mac some default (eps/web) might be kept on None. This is not 838 #suitable for LINUX which doesn't have open command. 839 840 # first for eps_viewer 841 if not cls.eps_viewer: 842 cls.eps_viewer = cls.find_valid(['evince','gv', 'ggv'], 'eps viewer') 843 844 # Second for web browser 845 if not cls.web_browser: 846 cls.web_browser = cls.find_valid( 847 ['firefox', 'chrome', 'safari','opera'], 848 'web browser')
849 850 @classmethod
851 - def configure_mac(cls, configuration=None):
852 """ configure the way to open a file for mac """ 853 854 if configuration is None: 855 configuration = {'text_editor': None, 856 'eps_viewer':None, 857 'web_browser':None} 858 859 for key in configuration: 860 if key == 'text_editor': 861 # Treat text editor ONLY text base editor !! 862 if configuration[key]: 863 program = configuration[key].split()[0] 864 if not which(program): 865 logger.warning('Specified text editor %s not valid.' % \ 866 configuration[key]) 867 else: 868 # All is good 869 cls.text_editor = configuration[key] 870 continue 871 #Need to find a valid default 872 if os.environ.has_key('EDITOR'): 873 cls.text_editor = os.environ['EDITOR'] 874 else: 875 cls.text_editor = cls.find_valid( 876 ['vi', 'emacs', 'vim', 'gedit', 'nano'], 877 'text editor') 878 879 elif key == 'eps_viewer': 880 if configuration[key]: 881 cls.eps_viewer = configuration[key] 882 continue 883 # else keep None. For Mac this will use the open command. 884 elif key == 'web_browser': 885 if configuration[key]: 886 cls.web_browser = configuration[key] 887 continue
888 # else keep None. For Mac this will use the open command. 889 890 @staticmethod
891 - def find_valid(possibility, program='program'):
892 """find a valid shell program in the list""" 893 894 for p in possibility: 895 if which(p): 896 logger.info('Using default %s \"%s\". ' % (program, p) + \ 897 'Set another one in ./input/mg5_configuration.txt') 898 return p 899 900 logger.info('No valid %s found. ' % program + \ 901 'Please set in ./input/mg5_configuration.txt') 902 return None
903 904
905 - def open_program(self, program, file_path, mac_check=True, background=False):
906 """ open a file with a given program """ 907 908 if mac_check==True and sys.platform == 'darwin': 909 return self.open_mac_program(program, file_path) 910 911 # Shell program only 912 if program: 913 arguments = program.split() # allow argument in program definition 914 arguments.append(file_path) 915 916 if not background: 917 subprocess.call(arguments) 918 else: 919 import thread 920 thread.start_new_thread(subprocess.call,(arguments,)) 921 else: 922 logger.warning('Not able to open file %s since no program configured.' % file_path + \ 923 'Please set one in ./input/mg5_configuration.txt')
924
925 - def open_mac_program(self, program, file_path):
926 """ open a text with the text editor """ 927 928 if not program: 929 # Ask to mac manager 930 os.system('open %s' % file_path) 931 elif which(program): 932 # shell program 933 arguments = program.split() # Allow argument in program definition 934 arguments.append(file_path) 935 subprocess.call(arguments) 936 else: 937 # not shell program 938 os.system('open -a %s %s' % (program, file_path))
939
940 -def is_executable(path):
941 """ check if a path is executable""" 942 try: 943 return os.access(path, os.X_OK) 944 except Exception: 945 return False
946
947 -class OptionParser(optparse.OptionParser):
948 """Option Peaser which raise an error instead as calling exit""" 949
950 - def exit(self, status=0, msg=None):
951 if msg: 952 raise InvalidCmd, msg 953 else: 954 raise InvalidCmd
955
956 -def sprint(*args, **opt):
957 """Returns the current line number in our program.""" 958 959 if not __debug__: 960 return 961 962 import inspect 963 if opt.has_key('log'): 964 log = opt['log'] 965 else: 966 log = logging.getLogger('madgraph') 967 if opt.has_key('level'): 968 level = opt['level'] 969 else: 970 level = logging.getLogger('madgraph').level 971 #print "madgraph level",level 972 #if level == 20: 973 # level = 10 #avoid info level 974 #print "use", level 975 lineno = inspect.currentframe().f_back.f_lineno 976 fargs = inspect.getframeinfo(inspect.currentframe().f_back) 977 filename, lineno = fargs[:2] 978 #file = inspect.currentframe().f_back.co_filename 979 #print type(file) 980 try: 981 source = inspect.getsourcelines(inspect.currentframe().f_back) 982 line = source[0][lineno-source[1]] 983 line = re.findall(r"misc\.sprint\(\s*(.*)\)\s*($|#)", line)[0][0] 984 if line.startswith("'") and line.endswith("'") and line.count(",") ==0: 985 line= '' 986 elif line.startswith("\"") and line.endswith("\"") and line.count(",") ==0: 987 line= '' 988 elif line.startswith(("\"","'")) and len(args)==1 and "%" in line: 989 line= '' 990 except Exception: 991 line='' 992 993 if line: 994 intro = ' %s = \033[0m' % line 995 else: 996 intro = '' 997 998 999 log.log(level, ' '.join([intro]+[str(a) for a in args]) + \ 1000 ' \033[1;30m[%s at line %s]\033[0m' % (os.path.basename(filename), lineno)) 1001 return
1002
1003 ################################################################################ 1004 # function to check if two float are approximatively equal 1005 ################################################################################ 1006 -def equal(a,b,sig_fig=6, zero_limit=True):
1007 """function to check if two float are approximatively equal""" 1008 import math 1009 1010 if not a or not b: 1011 if zero_limit: 1012 power = sig_fig + 1 1013 else: 1014 return a == b 1015 else: 1016 power = sig_fig - int(math.log10(abs(a))) + 1 1017 1018 return ( a==b or abs(int(a*10**power) - int(b*10**power)) < 10)
1019
1020 ################################################################################ 1021 # class to change directory with the "with statement" 1022 ################################################################################ 1023 -class chdir:
1024 - def __init__(self, newPath):
1025 self.newPath = newPath
1026
1027 - def __enter__(self):
1028 self.savedPath = os.getcwd() 1029 os.chdir(self.newPath)
1030
1031 - def __exit__(self, etype, value, traceback):
1032 os.chdir(self.savedPath)
1033
1034 1035 1036 ################################################################################ 1037 # TAIL FUNCTION 1038 ################################################################################ 1039 -class digest:
1040
1041 - def test_all(self):
1042 try: 1043 return self.test_hashlib() 1044 except Exception: 1045 pass 1046 try: 1047 return self.test_md5() 1048 except Exception: 1049 pass 1050 try: 1051 return self.test_zlib() 1052 except Exception: 1053 pass
1054
1055 - def test_hashlib(self):
1056 import hashlib 1057 def digest(text): 1058 """using mg5 for the hash""" 1059 t = hashlib.md5() 1060 t.update(text) 1061 return t.hexdigest()
1062 return digest
1063
1064 - def test_md5(self):
1065 import md5 1066 def digest(text): 1067 """using mg5 for the hash""" 1068 t = md5.md5() 1069 t.update(text) 1070 return t.hexdigest()
1071 return digest 1072
1073 - def test_zlib(self):
1074 import zlib 1075 def digest(text): 1076 return zlib.adler32(text)
1077 1078 digest = digest().test_all()
1079 1080 #=============================================================================== 1081 # Helper class for timing and RAM flashing of subprocesses. 1082 #=============================================================================== 1083 -class ProcessTimer:
1084 - def __init__(self,*args,**opts):
1085 self.cmd_args = args 1086 self.cmd_opts = opts 1087 self.execution_state = False
1088
1089 - def execute(self):
1090 self.max_vms_memory = 0 1091 self.max_rss_memory = 0 1092 1093 self.t1 = None 1094 self.t0 = time.time() 1095 self.p = subprocess.Popen(*self.cmd_args,**self.cmd_opts) 1096 self.execution_state = True
1097
1098 - def poll(self):
1099 if not self.check_execution_state(): 1100 return False 1101 1102 self.t1 = time.time() 1103 flash = subprocess.Popen("ps -p %i -o rss"%self.p.pid, 1104 shell=True,stdout=subprocess.PIPE) 1105 stdout_list = flash.communicate()[0].split('\n') 1106 rss_memory = int(stdout_list[1]) 1107 # for now we ignore vms 1108 vms_memory = 0 1109 1110 # This is the neat version using psutil 1111 # try: 1112 # pp = psutil.Process(self.p.pid) 1113 # 1114 # # obtain a list of the subprocess and all its descendants 1115 # descendants = list(pp.get_children(recursive=True)) 1116 # descendants = descendants + [pp] 1117 # 1118 # rss_memory = 0 1119 # vms_memory = 0 1120 # 1121 # # calculate and sum up the memory of the subprocess and all its descendants 1122 # for descendant in descendants: 1123 # try: 1124 # mem_info = descendant.get_memory_info() 1125 # 1126 # rss_memory += mem_info[0] 1127 # vms_memory += mem_info[1] 1128 # except psutil.error.NoSuchProcess: 1129 # # sometimes a subprocess descendant will have terminated between the time 1130 # # we obtain a list of descendants, and the time we actually poll this 1131 # # descendant's memory usage. 1132 # pass 1133 # 1134 # except psutil.error.NoSuchProcess: 1135 # return self.check_execution_state() 1136 1137 self.max_vms_memory = max(self.max_vms_memory,vms_memory) 1138 self.max_rss_memory = max(self.max_rss_memory,rss_memory) 1139 1140 return self.check_execution_state()
1141
1142 - def is_running(self):
1143 # Version with psutil 1144 # return psutil.pid_exists(self.p.pid) and self.p.poll() == None 1145 return self.p.poll() == None
1146
1147 - def check_execution_state(self):
1148 if not self.execution_state: 1149 return False 1150 if self.is_running(): 1151 return True 1152 self.executation_state = False 1153 self.t1 = time.time() 1154 return False
1155
1156 - def close(self,kill=False):
1157 1158 if self.p.poll() == None: 1159 if kill: 1160 self.p.kill() 1161 else: 1162 self.p.terminate()
1163 1164 # Again a neater handling with psutil 1165 # try: 1166 # pp = psutil.Process(self.p.pid) 1167 # if kill: 1168 # pp.kill() 1169 # else: 1170 # pp.terminate() 1171 # except psutil.error.NoSuchProcess: 1172 # pass 1173