Package madgraph :: Package madevent :: Module gen_ximprove
[hide private]
[frames] | no frames]

Source Code for Module madgraph.madevent.gen_ximprove

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2014 The MadGraph5_aMC@NLO Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph5_aMC@NLO project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph5_aMC@NLO license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, visit madgraph.phys.ucl.ac.be and amcatnlo.web.cern.ch 
  13  # 
  14  ################################################################################ 
  15  """ A python file to replace the fortran script gen_ximprove. 
  16      This script analyses the result of the survey/ previous refine and  
  17      creates the jobs for the following script. 
  18  """ 
  19  from __future__ import division 
  20   
  21  from __future__ import absolute_import 
  22  import collections 
  23  import os 
  24  import glob 
  25  import logging 
  26  import math 
  27  import re 
  28  import subprocess 
  29  import shutil 
  30  import stat 
  31  import sys 
  32  import six 
  33  from six.moves import range 
  34  from six.moves import zip 
  35   
  36  try: 
  37      import madgraph 
  38  except ImportError: 
  39      MADEVENT = True 
  40      import internal.sum_html as sum_html 
  41      import internal.banner as bannermod 
  42      import internal.misc as misc 
  43      import internal.files as files 
  44      import internal.cluster as cluster 
  45      import internal.combine_grid as combine_grid 
  46      import internal.combine_runs as combine_runs 
  47      import internal.lhe_parser as lhe_parser 
  48      if six.PY3: 
  49          import internal.hel_recycle as hel_recycle 
  50  else: 
  51      MADEVENT= False 
  52      import madgraph.madevent.sum_html as sum_html 
  53      import madgraph.various.banner as bannermod 
  54      import madgraph.various.misc as misc 
  55      import madgraph.iolibs.files as files 
  56      import madgraph.various.cluster as cluster 
  57      import madgraph.madevent.combine_grid as combine_grid 
  58      import madgraph.madevent.combine_runs as combine_runs 
  59      import madgraph.various.lhe_parser as lhe_parser 
  60      if six.PY3: 
  61          import madgraph.madevent.hel_recycle as hel_recycle 
  62   
  63  logger = logging.getLogger('madgraph.madevent.gen_ximprove') 
  64  pjoin = os.path.join 
65 66 -class gensym(object):
67 """a class to call the fortran gensym executable and handle it's output 68 in order to create the various job that are needed for the survey""" 69 70 #convenient shortcut for the formatting of variable 71 @ staticmethod
72 - def format_variable(*args):
73 return bannermod.ConfigFile.format_variable(*args)
74 75 combining_job = 2 # number of channel by ajob 76 splitted_grid = False 77 min_iterations = 3 78 mode= "survey" 79 80
81 - def __init__(self, cmd, opt=None):
82 83 try: 84 super(gensym, self).__init__(cmd, opt) 85 except TypeError: 86 pass 87 88 # Run statistics, a dictionary of RunStatistics(), with 89 self.run_statistics = {} 90 91 self.cmd = cmd 92 self.run_card = cmd.run_card 93 self.me_dir = cmd.me_dir 94 95 96 # dictionary to keep track of the precision when combining iteration 97 self.cross = collections.defaultdict(int) 98 self.abscross = collections.defaultdict(int) 99 self.sigma = collections.defaultdict(int) 100 self.chi2 = collections.defaultdict(int) 101 102 self.splitted_grid = False 103 if self.cmd.proc_characteristics['loop_induced']: 104 nexternal = self.cmd.proc_characteristics['nexternal'] 105 self.splitted_grid = max(2, (nexternal-2)**2) 106 if hasattr(self.cmd, "opts") and self.cmd.opts['accuracy'] == 0.1: 107 self.cmd.opts['accuracy'] = 0.02 108 109 if isinstance(cmd.cluster, cluster.MultiCore) and self.splitted_grid > 1: 110 self.splitted_grid = int(cmd.cluster.nb_core**0.5) 111 if self.splitted_grid == 1 and cmd.cluster.nb_core >1: 112 self.splitted_grid = 2 113 114 #if the user defines it in the run_card: 115 if self.run_card['survey_splitting'] != -1: 116 self.splitted_grid = self.run_card['survey_splitting'] 117 if self.run_card['survey_nchannel_per_job'] != 1 and 'survey_nchannel_per_job' in self.run_card.user_set: 118 self.combining_job = self.run_card['survey_nchannel_per_job'] 119 elif self.run_card['hard_survey'] > 1: 120 self.combining_job = 1 121 122 123 self.splitted_Pdir = {} 124 self.splitted_for_dir = lambda x,y: self.splitted_grid 125 self.combining_job_for_Pdir = lambda x: self.combining_job 126 self.lastoffset = {}
127 128 done_warning_zero_coupling = False
129 - def get_helicity(self, to_submit=True, clean=True):
130 """launch a single call to madevent to get the list of non zero helicity""" 131 132 self.subproc = [l.strip() for l in open(pjoin(self.me_dir,'SubProcesses', 133 'subproc.mg'))] 134 subproc = self.subproc 135 P_zero_result = [] 136 nb_tot_proc = len(subproc) 137 job_list = {} 138 139 140 for nb_proc,subdir in enumerate(subproc): 141 self.cmd.update_status('Compiling for process %s/%s.' % \ 142 (nb_proc+1,nb_tot_proc), level=None) 143 144 subdir = subdir.strip() 145 Pdir = pjoin(self.me_dir, 'SubProcesses',subdir) 146 logger.info(' %s ' % subdir) 147 148 self.cmd.compile(['madevent_forhel'], cwd=Pdir) 149 if not os.path.exists(pjoin(Pdir, 'madevent_forhel')): 150 raise Exception('Error make madevent_forhel not successful') 151 152 if not os.path.exists(pjoin(Pdir, 'Hel')): 153 os.mkdir(pjoin(Pdir, 'Hel')) 154 ff = open(pjoin(Pdir, 'Hel', 'input_app.txt'),'w') 155 ff.write('1000 1 1 \n 0.1 \n 2\n 0\n -1\n 1\n') 156 ff.close() 157 else: 158 try: 159 os.remove(pjoin(Pdir, 'Hel','results.dat')) 160 except Exception: 161 pass 162 # Launch gensym 163 p = misc.Popen(['../madevent_forhel < input_app.txt'], stdout=subprocess.PIPE, 164 stderr=subprocess.STDOUT, cwd=pjoin(Pdir,'Hel'), shell=True) 165 #sym_input = "%(points)d %(iterations)d %(accuracy)f \n" % self.opts 166 (stdout, _) = p.communicate(" ".encode()) 167 stdout = stdout.decode('ascii') 168 if os.path.exists(pjoin(self.me_dir,'error')): 169 raise Exception(pjoin(self.me_dir,'error')) 170 # note a continue is not enough here, we have in top to link 171 # the matrixX_optim.f to matrixX_orig.f to let the code to work 172 # after this error. 173 174 if 'no events passed cuts' in stdout: 175 raise Exception 176 177 all_zamp = set() 178 all_hel = set() 179 zero_gc = list() 180 all_zampperhel = set() 181 all_bad_amps_perhel = set() 182 183 for line in stdout.splitlines(): 184 if 'GC_' in line: 185 lsplit = line.split() 186 if float(lsplit[2]) ==0 == float(lsplit[3]): 187 zero_gc.append(lsplit[0]) 188 if 'Matrix Element/Good Helicity:' in line: 189 all_hel.add(tuple(line.split()[3:5])) 190 if 'Amplitude/ZEROAMP:' in line: 191 all_zamp.add(tuple(line.split()[1:3])) 192 if 'HEL/ZEROAMP:' in line: 193 nb_mat, nb_hel, nb_amp = line.split()[1:4] 194 if (nb_mat, nb_hel) not in all_hel: 195 continue 196 if (nb_mat,nb_amp) in all_zamp: 197 continue 198 all_zampperhel.add(tuple(line.split()[1:4])) 199 200 if zero_gc and not gensym.done_warning_zero_coupling: 201 gensym.done_warning_zero_coupling = True 202 logger.warning("The optimizer detects that you have coupling evaluated to zero: \n"+\ 203 "%s\n" % (' '.join(zero_gc)) +\ 204 "This will slow down the computation. Please consider using restricted model:\n" +\ 205 "https://answers.launchpad.net/mg5amcnlo/+faq/2312") 206 207 208 all_good_hels = collections.defaultdict(list) 209 for me_index, hel in all_hel: 210 all_good_hels[me_index].append(int(hel)) 211 212 #print(all_hel) 213 if self.run_card['hel_zeroamp']: 214 all_bad_amps = collections.defaultdict(list) 215 for me_index, amp in all_zamp: 216 all_bad_amps[me_index].append(int(amp)) 217 218 all_bad_amps_perhel = collections.defaultdict(list) 219 for me_index, hel, amp in all_zampperhel: 220 all_bad_amps_perhel[me_index].append((int(hel),int(amp))) 221 222 elif all_zamp: 223 nb_zero = sum(int(a[1]) for a in all_zamp) 224 if zero_gc: 225 logger.warning("Those zero couplings lead to %s Feynman diagram evaluated to zero (on 10 PS point),\n" % nb_zero +\ 226 "This part can optimize if you set the flag hel_zeroamp to True in the run_card."+\ 227 "Note that restricted model will be more optimal.") 228 else: 229 logger.warning("The optimization detected that you have %i zero matrix-element for this SubProcess: %s.\n" % nb_zero +\ 230 "This part can optimize if you set the flag hel_zeroamp to True in the run_card.") 231 232 #check if we need to do something and write associate information" 233 data = [all_hel, all_zamp, all_bad_amps_perhel] 234 if not self.run_card['hel_zeroamp']: 235 data[1] = '' 236 if not self.run_card['hel_filtering']: 237 data[0] = '' 238 data = str(data) 239 if os.path.exists(pjoin(Pdir,'Hel','selection')): 240 old_data = open(pjoin(Pdir,'Hel','selection')).read() 241 if old_data == data: 242 continue 243 244 245 with open(pjoin(Pdir,'Hel','selection'),'w') as fsock: 246 fsock.write(data) 247 248 249 for matrix_file in misc.glob('matrix*orig.f', Pdir): 250 251 split_file = matrix_file.split('/') 252 me_index = split_file[-1][len('matrix'):-len('_orig.f')] 253 254 basename = split_file[-1].replace('orig', 'optim') 255 split_out = split_file[:-1] + [basename] 256 out_file = pjoin('/', '/'.join(split_out)) 257 258 basename = 'template_%s' % split_file[-1].replace("_orig", "") 259 split_templ = split_file[:-1] + [basename] 260 templ_file = pjoin('/', '/'.join(split_templ)) 261 262 # Convert to sorted list for reproducibility 263 #good_hels = sorted(list(good_hels)) 264 good_hels = [str(x) for x in sorted(all_good_hels[me_index])] 265 if self.run_card['hel_zeroamp']: 266 267 bad_amps = [str(x) for x in sorted(all_bad_amps[me_index])] 268 bad_amps_perhel = [x for x in sorted(all_bad_amps_perhel[me_index])] 269 else: 270 bad_amps = [] 271 bad_amps_perhel = [] 272 if __debug__: 273 mtext = open(matrix_file).read() 274 nb_amp = int(re.findall('PARAMETER \(NGRAPHS=(\d+)\)', mtext)[0]) 275 logger.debug('nb_hel: %s zero amp: %s bad_amps_hel: %s/%s', len(good_hels),len(bad_amps),len(bad_amps_perhel), len(good_hels)*nb_amp ) 276 if len(good_hels) == 1: 277 files.cp(matrix_file, matrix_file.replace('orig','optim')) 278 continue # avoid optimization if onlye one helicity 279 recycler = hel_recycle.HelicityRecycler(good_hels, bad_amps, bad_amps_perhel) 280 # In case of bugs you can play around with these: 281 recycler.hel_filt = self.run_card['hel_filtering'] 282 recycler.amp_splt = self.run_card['hel_splitamp'] 283 recycler.amp_filt = self.run_card['hel_zeroamp'] 284 285 recycler.set_input(matrix_file) 286 recycler.set_output(out_file) 287 recycler.set_template(templ_file) 288 recycler.generate_output_file() 289 del recycler 290 291 # with misc.chdir(): 292 # pass 293 294 #files.ln(pjoin(Pdir, 'madevent_forhel'), Pdir, name='madevent') ##to be removed 295 296 return {}, P_zero_result
297 298
299 - def launch(self, to_submit=True, clean=True):
300 """ """ 301 302 if not hasattr(self, 'subproc'): 303 self.subproc = [l.strip() for l in open(pjoin(self.me_dir,'SubProcesses', 304 'subproc.mg'))] 305 subproc = self.subproc 306 307 P_zero_result = [] # check the number of times where they are no phase-space 308 309 nb_tot_proc = len(subproc) 310 job_list = {} 311 for nb_proc,subdir in enumerate(subproc): 312 self.cmd.update_status('Compiling for process %s/%s. <br> (previous processes already running)' % \ 313 (nb_proc+1,nb_tot_proc), level=None) 314 315 subdir = subdir.strip() 316 Pdir = pjoin(self.me_dir, 'SubProcesses',subdir) 317 logger.info(' %s ' % subdir) 318 319 # clean previous run 320 if clean: 321 for match in misc.glob('*ajob*', Pdir): 322 if os.path.basename(match)[:4] in ['ajob', 'wait', 'run.', 'done']: 323 os.remove(match) 324 for match in misc.glob('G*', Pdir): 325 if os.path.exists(pjoin(match,'results.dat')): 326 os.remove(pjoin(match, 'results.dat')) 327 if os.path.exists(pjoin(match, 'ftn25')): 328 os.remove(pjoin(match, 'ftn25')) 329 330 #compile gensym 331 self.cmd.compile(['gensym'], cwd=Pdir) 332 if not os.path.exists(pjoin(Pdir, 'gensym')): 333 raise Exception('Error make gensym not successful') 334 335 # Launch gensym 336 p = misc.Popen(['./gensym'], stdout=subprocess.PIPE, 337 stderr=subprocess.STDOUT, cwd=Pdir) 338 #sym_input = "%(points)d %(iterations)d %(accuracy)f \n" % self.opts 339 (stdout, _) = p.communicate(''.encode()) 340 stdout = stdout.decode('ascii') 341 if os.path.exists(pjoin(self.me_dir,'error')): 342 files.mv(pjoin(self.me_dir,'error'), pjoin(Pdir,'ajob.no_ps.log')) 343 P_zero_result.append(subdir) 344 continue 345 346 jobs = stdout.split() 347 job_list[Pdir] = jobs 348 try: 349 # check that all input are valid 350 [float(s) for s in jobs] 351 except Exception: 352 logger.debug("unformated string found in gensym. Please check:\n %s" % stdout) 353 done=False 354 job_list[Pdir] = [] 355 lines = stdout.split('\n') 356 for l in lines: 357 try: 358 [float(s) for s in l.split()] 359 except: 360 continue 361 else: 362 if done: 363 raise Exception('Parsing error in gensym: %s' % stdout) 364 job_list[Pdir] = l.split() 365 done = True 366 if not done: 367 raise Exception('Parsing error in gensym: %s' % stdout) 368 369 self.cmd.compile(['madevent'], cwd=Pdir) 370 if to_submit: 371 self.submit_to_cluster(job_list) 372 job_list = {} 373 374 return job_list, P_zero_result
375
376 - def resubmit(self, min_precision=1.0, resubmit_zero=False):
377 """collect the result of the current run and relaunch each channel 378 not completed or optionally a completed one with a precision worse than 379 a threshold (and/or the zero result channel)""" 380 381 job_list, P_zero_result = self.launch(to_submit=False, clean=False) 382 383 for P , jobs in dict(job_list).items(): 384 misc.sprint(jobs) 385 to_resub = [] 386 for job in jobs: 387 if os.path.exists(pjoin(P, 'G%s' % job)) and os.path.exists(pjoin(P, 'G%s' % job, 'results.dat')): 388 one_result = sum_html.OneResult(job) 389 try: 390 one_result.read_results(pjoin(P, 'G%s' % job, 'results.dat')) 391 except: 392 to_resub.append(job) 393 if one_result.xsec == 0: 394 if resubmit_zero: 395 to_resub.append(job) 396 elif max(one_result.xerru, one_result.xerrc)/one_result.xsec > min_precision: 397 to_resub.append(job) 398 else: 399 to_resub.append(job) 400 if to_resub: 401 for G in to_resub: 402 try: 403 shutil.rmtree(pjoin(P, 'G%s' % G)) 404 except Exception as error: 405 misc.sprint(error) 406 pass 407 misc.sprint(to_resub) 408 self.submit_to_cluster({P: to_resub})
409 410 411 412 413 414 415 416 417 418 419
420 - def submit_to_cluster(self, job_list):
421 """ """ 422 423 if self.run_card['job_strategy'] > 0: 424 if len(job_list) >1: 425 for path, dirs in job_list.items(): 426 self.submit_to_cluster({path:dirs}) 427 return 428 path, value = list(job_list.items())[0] 429 nexternal = self.cmd.proc_characteristics['nexternal'] 430 current = open(pjoin(path, "nexternal.inc")).read() 431 ext = re.search(r"PARAMETER \(NEXTERNAL=(\d+)\)", current).group(1) 432 433 if self.run_card['job_strategy'] == 2: 434 self.splitted_grid = 2 435 if nexternal == int(ext): 436 to_split = 2 437 else: 438 to_split = 0 439 if hasattr(self, 'splitted_Pdir'): 440 self.splitted_Pdir[path] = to_split 441 else: 442 self.splitted_Pdir = {path: to_split} 443 self.splitted_for_dir = lambda x,y : self.splitted_Pdir[x] 444 elif self.run_card['job_strategy'] == 1: 445 if nexternal == int(ext): 446 combine = 1 447 else: 448 combine = self.combining_job 449 if hasattr(self, 'splitted_Pdir'): 450 self.splitted_Pdir[path] = combine 451 else: 452 self.splitted_Pdir = {path: combine} 453 self.combining_job_for_Pdir = lambda x : self.splitted_Pdir[x] 454 455 if not self.splitted_grid: 456 return self.submit_to_cluster_no_splitting(job_list) 457 elif self.cmd.cluster_mode == 0: 458 return self.submit_to_cluster_no_splitting(job_list) 459 elif self.cmd.cluster_mode == 2 and self.cmd.options['nb_core'] == 1: 460 return self.submit_to_cluster_no_splitting(job_list) 461 else: 462 return self.submit_to_cluster_splitted(job_list)
463 464
465 - def submit_to_cluster_no_splitting(self, job_list):
466 """submit the survey without the parralelization. 467 This is the old mode which is still usefull in single core""" 468 469 # write the template file for the parameter file 470 self.write_parameter(parralelization=False, Pdirs=list(job_list.keys())) 471 472 473 # launch the job with the appropriate grouping 474 for Pdir, jobs in job_list.items(): 475 jobs = list(jobs) 476 i=0 477 while jobs: 478 i+=1 479 to_submit = ['0'] # the first entry is actually the offset 480 for _ in range(self.combining_job_for_Pdir(Pdir)): 481 if jobs: 482 to_submit.append(jobs.pop(0)) 483 484 self.cmd.launch_job(pjoin(self.me_dir, 'SubProcesses', 'survey.sh'), 485 argument=to_submit, 486 cwd=pjoin(self.me_dir,'SubProcesses' , Pdir))
487 488
489 - def create_resubmit_one_iter(self, Pdir, G, submit_ps, nb_job, step=0):
490 """prepare the input_file for submitting the channel""" 491 492 493 if 'SubProcesses' not in Pdir: 494 Pdir = pjoin(self.me_dir, 'SubProcesses', Pdir) 495 496 #keep track of how many job are sended 497 self.splitted_Pdir[(Pdir, G)] = int(nb_job) 498 499 500 # 1. write the new input_app.txt 501 run_card = self.cmd.run_card 502 options = {'event' : submit_ps, 503 'maxiter': 1, 504 'miniter': 1, 505 'accuracy': self.cmd.opts['accuracy'], 506 'helicity': run_card['nhel_survey'] if 'nhel_survey' in run_card \ 507 else run_card['nhel'], 508 'gridmode': -2, 509 'channel' : G 510 } 511 512 Gdir = pjoin(Pdir, 'G%s' % G) 513 self.write_parameter_file(pjoin(Gdir, 'input_app.txt'), options) 514 515 # 2. check that ftn25 exists. 516 assert os.path.exists(pjoin(Gdir, "ftn25")) 517 518 519 # 3. Submit the new jobs 520 #call back function 521 packet = cluster.Packet((Pdir, G, step+1), 522 self.combine_iteration, 523 (Pdir, G, step+1)) 524 525 if step ==0: 526 self.lastoffset[(Pdir, G)] = 0 527 528 # resubmit the new jobs 529 for i in range(int(nb_job)): 530 name = "G%s_%s" % (G,i+1) 531 self.lastoffset[(Pdir, G)] += 1 532 offset = self.lastoffset[(Pdir, G)] 533 self.cmd.launch_job(pjoin(self.me_dir, 'SubProcesses', 'refine_splitted.sh'), 534 argument=[name, 'G%s'%G, offset], 535 cwd= Pdir, 536 packet_member=packet)
537 538
539 - def submit_to_cluster_splitted(self, job_list):
540 """ submit the version of the survey with splitted grid creation 541 """ 542 543 #if self.splitted_grid <= 1: 544 # return self.submit_to_cluster_no_splitting(job_list) 545 546 for Pdir, jobs in job_list.items(): 547 if not jobs: 548 continue 549 if self.splitted_for_dir(Pdir, jobs[0]) <= 1: 550 return self.submit_to_cluster_no_splitting({Pdir:jobs}) 551 552 self.write_parameter(parralelization=True, Pdirs=[Pdir]) 553 # launch the job with the appropriate grouping 554 555 for job in jobs: 556 packet = cluster.Packet((Pdir, job, 1), self.combine_iteration, (Pdir, job, 1)) 557 for i in range(self.splitted_for_dir(Pdir, job)): 558 self.cmd.launch_job(pjoin(self.me_dir, 'SubProcesses', 'survey.sh'), 559 argument=[i+1, job], 560 cwd=pjoin(self.me_dir,'SubProcesses' , Pdir), 561 packet_member=packet)
562
563 - def combine_iteration(self, Pdir, G, step):
564 565 grid_calculator, cross, error = self.combine_grid(Pdir, G, step) 566 567 # Compute the number of events used for this run. 568 nb_events = grid_calculator.target_evt 569 570 Gdirs = [] #build the the list of directory 571 for i in range(self.splitted_for_dir(Pdir, G)): 572 path = pjoin(Pdir, "G%s_%s" % (G, i+1)) 573 Gdirs.append(path) 574 575 # 4. make the submission of the next iteration 576 # Three cases - less than 3 iteration -> continue 577 # - more than 3 and less than 5 -> check error 578 # - more than 5 -> prepare info for refine 579 need_submit = False 580 if step < self.min_iterations and cross != 0: 581 if step == 1: 582 need_submit = True 583 else: 584 across = self.abscross[(Pdir,G)]/(self.sigma[(Pdir,G)]+1e-99) 585 tot_across = self.get_current_axsec() 586 if across / tot_across < 1e-6: 587 need_submit = False 588 elif error < self.cmd.opts['accuracy'] / 100: 589 need_submit = False 590 else: 591 need_submit = True 592 593 elif step >= self.cmd.opts['iterations']: 594 need_submit = False 595 elif self.cmd.opts['accuracy'] < 0: 596 #check for luminosity 597 raise Exception("Not Implemented") 598 elif self.abscross[(Pdir,G)] == 0: 599 need_submit = False 600 else: 601 across = self.abscross[(Pdir,G)]/(self.sigma[(Pdir,G)]+1e-99) 602 tot_across = self.get_current_axsec() 603 if across == 0: 604 need_submit = False 605 elif across / tot_across < 1e-5: 606 need_submit = False 607 elif error > self.cmd.opts['accuracy']: 608 need_submit = True 609 else: 610 need_submit = False 611 612 613 if cross: 614 grid_calculator.write_grid_for_submission(Pdir,G, 615 self.splitted_for_dir(Pdir, G), 616 nb_events,mode=self.mode, 617 conservative_factor=5.0) 618 619 xsec_format = '.%ig'%(max(3,int(math.log10(1.0/float(error)))+2) 620 if float(cross)!=0.0 and float(error)!=0.0 else 8) 621 if need_submit: 622 message = "%%s/G%%s is at %%%s +- %%.3g pb. Now submitting iteration #%s."%(xsec_format, step+1) 623 logger.info(message%\ 624 (os.path.basename(Pdir), G, float(cross), 625 float(error)*float(cross))) 626 self.resubmit_survey(Pdir,G, Gdirs, step) 627 elif cross: 628 logger.info("Survey finished for %s/G%s at %s"%( 629 os.path.basename(Pdir),G,('%%%s +- %%.3g pb'%xsec_format))% 630 (float(cross), float(error)*float(cross))) 631 # prepare information for refine 632 newGpath = pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G) 633 if not os.path.exists(newGpath): 634 os.mkdir(newGpath) 635 636 # copy the new grid: 637 files.cp(pjoin(Gdirs[0], 'ftn25'), 638 pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G, 'ftn26')) 639 640 # copy the events 641 fsock = open(pjoin(newGpath, 'events.lhe'), 'w') 642 for Gdir in Gdirs: 643 fsock.write(open(pjoin(Gdir, 'events.lhe')).read()) 644 645 # copy one log 646 files.cp(pjoin(Gdirs[0], 'log.txt'), 647 pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G)) 648 649 650 # create the appropriate results.dat 651 self.write_results(grid_calculator, cross, error, Pdir, G, step) 652 else: 653 logger.info("Survey finished for %s/G%s [0 cross]", os.path.basename(Pdir),G) 654 655 Gdir = pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G) 656 if not os.path.exists(Gdir): 657 os.mkdir(Gdir) 658 # copy one log 659 files.cp(pjoin(Gdirs[0], 'log.txt'), Gdir) 660 # create the appropriate results.dat 661 self.write_results(grid_calculator, cross, error, Pdir, G, step) 662 663 return 0
664
665 - def combine_grid(self, Pdir, G, step, exclude_sub_jobs=[]):
666 """ exclude_sub_jobs is to remove some of the subjobs if a numerical 667 issue is detected in one of them. Warning is issue when this occurs. 668 """ 669 670 # 1. create an object to combine the grid information and fill it 671 grid_calculator = combine_grid.grid_information(self.run_card['nhel']) 672 673 for i in range(self.splitted_for_dir(Pdir, G)): 674 if i in exclude_sub_jobs: 675 continue 676 path = pjoin(Pdir, "G%s_%s" % (G, i+1)) 677 fsock = misc.mult_try_open(pjoin(path, 'results.dat')) 678 one_result = grid_calculator.add_results_information(fsock) 679 fsock.close() 680 if one_result.axsec == 0: 681 grid_calculator.onefail = True 682 continue # grid_information might not exists 683 fsock = misc.mult_try_open(pjoin(path, 'grid_information')) 684 grid_calculator.add_one_grid_information(fsock) 685 fsock.close() 686 os.remove(pjoin(path, 'results.dat')) 687 #os.remove(pjoin(path, 'grid_information')) 688 689 690 691 #2. combine the information about the total crossection / error 692 # start by keep the interation in memory 693 cross, across, sigma = grid_calculator.get_cross_section() 694 695 #3. Try to avoid one single PS point which ruins the integration 696 # Should be related to loop evaluation instability. 697 maxwgt = grid_calculator.get_max_wgt(0.01) 698 if maxwgt: 699 nunwgt = grid_calculator.get_nunwgt(maxwgt) 700 # Make sure not to apply the security below during the first step of the 701 # survey. Also, disregard channels with a contribution relative to the 702 # total cross-section smaller than 1e-8 since in this case it is unlikely 703 # that this channel will need more than 1 event anyway. 704 apply_instability_security = False 705 rel_contrib = 0.0 706 if (self.__class__ != gensym or step > 1): 707 Pdir_across = 0.0 708 Gdir_across = 0.0 709 for (mPdir,mG) in self.abscross.keys(): 710 if mPdir == Pdir: 711 Pdir_across += (self.abscross[(mPdir,mG)]/ 712 (self.sigma[(mPdir,mG)]+1e-99)) 713 if mG == G: 714 Gdir_across += (self.abscross[(mPdir,mG)]/ 715 (self.sigma[(mPdir,mG)]+1e-99)) 716 rel_contrib = abs(Gdir_across/(Pdir_across+1e-99)) 717 if rel_contrib > (1.0e-8) and \ 718 nunwgt < 2 and len(grid_calculator.results) > 1: 719 apply_instability_security = True 720 721 if apply_instability_security: 722 # check the ratio between the different submit 723 th_maxwgt = [(r.th_maxwgt,i) for i,r in enumerate(grid_calculator.results)] 724 th_maxwgt.sort() 725 ratio = th_maxwgt[-1][0]/th_maxwgt[-2][0] 726 if ratio > 1e4: 727 logger.warning( 728 """"One Event with large weight have been found (ratio = %.3g) in channel G%s (with rel.contrib=%.3g). 729 This is likely due to numerical instabilities. The associated job is discarded to recover. 730 For offline investigation, the problematic discarded events are stored in: 731 %s"""%(ratio,G,rel_contrib,pjoin(Pdir,'DiscardedUnstableEvents'))) 732 exclude_sub_jobs = list(exclude_sub_jobs) 733 exclude_sub_jobs.append(th_maxwgt[-1][1]) 734 grid_calculator.results.run_statistics['skipped_subchannel'] += 1 735 736 # Add some monitoring of the problematic events 737 gPath = pjoin(Pdir, "G%s_%s" % (G, th_maxwgt[-1][1]+1)) 738 if os.path.isfile(pjoin(gPath,'events.lhe')): 739 lhe_file = lhe_parser.EventFile(pjoin(gPath,'events.lhe')) 740 discardedPath = pjoin(Pdir,'DiscardedUnstableEvents') 741 if not os.path.exists(discardedPath): 742 os.mkdir(discardedPath) 743 if os.path.isdir(discardedPath): 744 # Keep only the event with a maximum weight, as it surely 745 # is the problematic one. 746 evtRecord = open(pjoin(discardedPath,'discarded_G%s.dat'%G),'a') 747 lhe_file.seek(0) #rewind the file 748 try: 749 evtRecord.write('\n'+str(max(lhe_file,key=lambda evt:abs(evt.wgt)))) 750 except Exception: 751 #something wrong write the full file. 752 lhe_file.close() 753 evtRecord.write(pjoin(gPath,'events.lhe').read()) 754 evtRecord.close() 755 756 return self.combine_grid(Pdir, G, step, exclude_sub_jobs) 757 758 759 if across !=0: 760 if sigma != 0: 761 self.cross[(Pdir,G)] += cross**3/sigma**2 762 self.abscross[(Pdir,G)] += across * cross**2/sigma**2 763 self.sigma[(Pdir,G)] += cross**2/ sigma**2 764 self.chi2[(Pdir,G)] += cross**4/sigma**2 765 # and use those iteration to get the current estimator 766 cross = self.cross[(Pdir,G)]/self.sigma[(Pdir,G)] 767 if step > 1: 768 error = math.sqrt(abs((self.chi2[(Pdir,G)]/cross**2 - \ 769 self.sigma[(Pdir,G)])/(step-1))/self.sigma[(Pdir,G)]) 770 else: 771 error = sigma/cross 772 else: 773 self.cross[(Pdir,G)] = cross 774 self.abscross[(Pdir,G)] = across 775 self.sigma[(Pdir,G)] = 0 776 self.chi2[(Pdir,G)] = 0 777 cross = self.cross[(Pdir,G)] 778 error = 0 779 780 else: 781 error = 0 782 783 grid_calculator.results.compute_values(update_statistics=True) 784 if (str(os.path.basename(Pdir)), G) in self.run_statistics: 785 self.run_statistics[(str(os.path.basename(Pdir)), G)]\ 786 .aggregate_statistics(grid_calculator.results.run_statistics) 787 else: 788 self.run_statistics[(str(os.path.basename(Pdir)), G)] = \ 789 grid_calculator.results.run_statistics 790 791 self.warnings_from_statistics(G, grid_calculator.results.run_statistics) 792 stats_msg = grid_calculator.results.run_statistics.nice_output( 793 '/'.join([os.path.basename(Pdir),'G%s'%G])) 794 795 if stats_msg: 796 logger.log(5, stats_msg) 797 798 # Clean up grid_information to avoid border effects in case of a crash 799 for i in range(self.splitted_for_dir(Pdir, G)): 800 path = pjoin(Pdir, "G%s_%s" % (G, i+1)) 801 try: 802 os.remove(pjoin(path, 'grid_information')) 803 except OSError as oneerror: 804 if oneerror.errno != 2: 805 raise 806 return grid_calculator, cross, error
807
808 - def warnings_from_statistics(self,G,stats):
809 """Possible warn user for worrying MadLoop stats for this channel.""" 810 811 if stats['n_madloop_calls']==0: 812 return 813 814 EPS_fraction = float(stats['exceptional_points'])/stats['n_madloop_calls'] 815 816 msg = "Channel %s has encountered a fraction of %.3g\n"+ \ 817 "of numerically unstable loop matrix element computations\n"+\ 818 "(which could not be rescued using quadruple precision).\n"+\ 819 "The results might not be trusted." 820 821 if 0.01 > EPS_fraction > 0.001: 822 logger.warning(msg%(G,EPS_fraction)) 823 elif EPS_fraction > 0.01: 824 logger.critical((msg%(G,EPS_fraction)).replace('might', 'can')) 825 raise Exception((msg%(G,EPS_fraction)).replace('might', 'can'))
826
827 - def get_current_axsec(self):
828 829 across = 0 830 for (Pdir,G) in self.abscross: 831 across += self.abscross[(Pdir,G)]/(self.sigma[(Pdir,G)]+1e-99) 832 return across
833
834 - def write_results(self, grid_calculator, cross, error, Pdir, G, step):
835 836 #compute the value 837 if cross == 0: 838 abscross,nw, luminosity = 0, 0, 0 839 wgt, maxit,nunwgt, wgt, nevents = 0,0,0,0,0 840 maxwgt = 0 841 error = 0 842 else: 843 grid_calculator.results.compute_values() 844 abscross = self.abscross[(Pdir,G)]/self.sigma[(Pdir,G)] 845 nw = grid_calculator.results.nw 846 wgt = grid_calculator.results.wgt 847 maxit = step 848 wgt = 0 849 nevents = grid_calculator.results.nevents 850 maxwgt = grid_calculator.get_max_wgt() 851 nunwgt = grid_calculator.get_nunwgt() 852 luminosity = nunwgt/cross 853 854 #format the results.dat 855 def fstr(nb): 856 data = '%E' % nb 857 nb, power = data.split('E') 858 nb = float(nb) /10 859 power = int(power) + 1 860 return '%.5fE%+03i' %(nb,power)
861 line = '%s %s %s %i %i %i %i %s %s %s %s 0.0 0\n' % \ 862 (fstr(cross), fstr(error*cross), fstr(error*cross), 863 nevents, nw, maxit,nunwgt, 864 fstr(luminosity), fstr(wgt), fstr(abscross), fstr(maxwgt)) 865 866 fsock = open(pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G, 867 'results.dat'),'w') 868 fsock.writelines(line) 869 fsock.close()
870
871 - def resubmit_survey(self, Pdir, G, Gdirs, step):
872 """submit the next iteration of the survey""" 873 874 # 1. write the new input_app.txt to double the number of points 875 run_card = self.cmd.run_card 876 options = {'event' : 2**(step) * self.cmd.opts['points'] / self.splitted_grid, 877 'maxiter': 1, 878 'miniter': 1, 879 'accuracy': self.cmd.opts['accuracy'], 880 'helicity': run_card['nhel_survey'] if 'nhel_survey' in run_card \ 881 else run_card['nhel'], 882 'gridmode': -2, 883 'channel' : '' 884 } 885 886 if int(options['helicity']) == 1: 887 options['event'] = options['event'] * 2**(self.cmd.proc_characteristics['nexternal']//3) 888 889 for Gdir in Gdirs: 890 self.write_parameter_file(pjoin(Gdir, 'input_app.txt'), options) 891 892 893 #2. resubmit the new jobs 894 packet = cluster.Packet((Pdir, G, step+1), self.combine_iteration, \ 895 (Pdir, G, step+1)) 896 nb_step = len(Gdirs) * (step+1) 897 for i,subdir in enumerate(Gdirs): 898 subdir = subdir.rsplit('_',1)[1] 899 subdir = int(subdir) 900 offset = nb_step+i+1 901 offset=str(offset) 902 tag = "%s.%s" % (subdir, offset) 903 904 self.cmd.launch_job(pjoin(self.me_dir, 'SubProcesses', 'survey.sh'), 905 argument=[tag, G], 906 cwd=pjoin(self.me_dir,'SubProcesses' , Pdir), 907 packet_member=packet)
908 909 910 911
912 - def write_parameter_file(self, path, options):
913 """ """ 914 915 template =""" %(event)s %(maxiter)s %(miniter)s !Number of events and max and min iterations 916 %(accuracy)s !Accuracy 917 %(gridmode)s !Grid Adjustment 0=none, 2=adjust 918 1 !Suppress Amplitude 1=yes 919 %(helicity)s !Helicity Sum/event 0=exact 920 %(channel)s """ 921 options['event'] = int(options['event']) 922 open(path, 'w').write(template % options)
923 924 925
926 - def write_parameter(self, parralelization, Pdirs=None):
927 """Write the parameter of the survey run""" 928 929 run_card = self.cmd.run_card 930 931 options = {'event' : self.cmd.opts['points'], 932 'maxiter': self.cmd.opts['iterations'], 933 'miniter': self.min_iterations, 934 'accuracy': self.cmd.opts['accuracy'], 935 'helicity': run_card['nhel_survey'] if 'nhel_survey' in run_card \ 936 else run_card['nhel'], 937 'gridmode': 2, 938 'channel': '' 939 } 940 941 if int(options['helicity'])== 1: 942 options['event'] = options['event'] * 2**(self.cmd.proc_characteristics['nexternal']//3) 943 944 if parralelization: 945 options['gridmode'] = -2 946 options['maxiter'] = 1 #this is automatic in dsample anyway 947 options['miniter'] = 1 #this is automatic in dsample anyway 948 options['event'] /= self.splitted_grid 949 950 if not Pdirs: 951 Pdirs = self.subproc 952 953 for Pdir in Pdirs: 954 path =pjoin(Pdir, 'input_app.txt') 955 self.write_parameter_file(path, options)
956
957 958 959 -class gen_ximprove(object):
960 961 962 # some hardcoded value which impact the generation 963 gen_events_security = 1.2 # multiply the number of requested event by this number for security 964 combining_job = 0 # allow to run multiple channel in sequence 965 max_request_event = 1000 # split jobs if a channel if it needs more than that 966 max_event_in_iter = 5000 967 min_event_in_iter = 1000 968 max_splitting = 130 # maximum duplication of a given channel 969 min_iter = 3 970 max_iter = 9 971 keep_grid_for_refine = False # only apply if needed to split the job 972 973 #convenient shortcut for the formatting of variable 974 @ staticmethod
975 - def format_variable(*args):
976 return bannermod.ConfigFile.format_variable(*args)
977 978
979 - def __new__(cls, cmd, opt):
980 """Choose in which type of refine we want to be""" 981 982 if cmd.proc_characteristics['loop_induced']: 983 return super(gen_ximprove, cls).__new__(gen_ximprove_share) 984 elif gen_ximprove.format_variable(cmd.run_card['gridpack'], bool): 985 return super(gen_ximprove, cls).__new__(gen_ximprove_gridpack) 986 elif cmd.run_card["job_strategy"] == 2: 987 return super(gen_ximprove, cls).__new__(gen_ximprove_share) 988 else: 989 return super(gen_ximprove, cls).__new__(gen_ximprove_v4)
990 991
992 - def __init__(self, cmd, opt=None):
993 994 try: 995 super(gen_ximprove, self).__init__(cmd, opt) 996 except TypeError: 997 pass 998 999 self.run_statistics = {} 1000 self.cmd = cmd 1001 self.run_card = cmd.run_card 1002 run_card = self.run_card 1003 self.me_dir = cmd.me_dir 1004 1005 #extract from the run_card the information that we need. 1006 self.gridpack = run_card['gridpack'] 1007 self.nhel = run_card['nhel'] 1008 if "nhel_refine" in run_card: 1009 self.nhel = run_card["nhel_refine"] 1010 1011 if self.run_card['refine_evt_by_job'] != -1: 1012 self.max_request_event = run_card['refine_evt_by_job'] 1013 1014 1015 # Default option for the run 1016 self.gen_events = True 1017 self.parralel = False 1018 # parameter which was input for the normal gen_ximprove run 1019 self.err_goal = 0.01 1020 self.max_np = 9 1021 self.split_channels = False 1022 # parameter for the gridpack run 1023 self.nreq = 2000 1024 self.iseed = 4321 1025 1026 # placeholder for information 1027 self.results = 0 #updated in launch/update_html 1028 1029 if isinstance(opt, dict): 1030 self.configure(opt) 1031 elif isinstance(opt, bannermod.GridpackCard): 1032 self.configure_gridpack(opt)
1033
1034 - def __call__(self):
1035 return self.launch()
1036
1037 - def launch(self):
1038 """running """ 1039 1040 #start the run 1041 self.handle_seed() 1042 self.results = sum_html.collect_result(self.cmd, 1043 main_dir=pjoin(self.cmd.me_dir,'SubProcesses')) #main_dir is for gridpack readonly mode 1044 if self.gen_events: 1045 # We run to provide a given number of events 1046 self.get_job_for_event() 1047 else: 1048 # We run to achieve a given precision 1049 self.get_job_for_precision()
1050 1051
1052 - def configure(self, opt):
1053 """Defines some parameter of the run""" 1054 1055 for key, value in opt.items(): 1056 if key in self.__dict__: 1057 targettype = type(getattr(self, key)) 1058 setattr(self, key, self.format_variable(value, targettype, key)) 1059 else: 1060 raise Exception('%s not define' % key) 1061 1062 1063 # special treatment always do outside the loop to avoid side effect 1064 if 'err_goal' in opt: 1065 if self.err_goal < 1: 1066 logger.info("running for accuracy %s%%" % (self.err_goal*100)) 1067 self.gen_events = False 1068 elif self.err_goal >= 1: 1069 logger.info("Generating %s unweighted events." % self.err_goal) 1070 self.gen_events = True 1071 self.err_goal = self.err_goal * self.gen_events_security # security
1072
1073 - def handle_seed(self):
1074 """not needed but for gridpack --which is not handle here for the moment""" 1075 return
1076 1077
1078 - def find_job_for_event(self):
1079 """return the list of channel that need to be improved""" 1080 1081 assert self.err_goal >=1 1082 self.err_goal = int(self.err_goal) 1083 1084 goal_lum = self.err_goal/(self.results.axsec+1e-99) #pb^-1 1085 logger.info('Effective Luminosity %s pb^-1', goal_lum) 1086 1087 all_channels = sum([list(P) for P in self.results],[]) 1088 all_channels.sort(key= lambda x:x.get('luminosity'), reverse=True) 1089 1090 to_refine = [] 1091 for C in all_channels: 1092 if C.get('axsec') == 0: 1093 continue 1094 if goal_lum/(C.get('luminosity')+1e-99) >= 1 + (self.gen_events_security-1)/2: 1095 logger.debug("channel %s need to improve by %.2f (xsec=%s pb, iter=%s)", C.name, goal_lum/(C.get('luminosity')+1e-99), C.get('xsec'), int(C.get('maxit'))) 1096 to_refine.append(C) 1097 elif C.get('xerr') > max(C.get('axsec'), 1098 (1/(100*math.sqrt(self.err_goal)))*all_channels[-1].get('axsec')): 1099 to_refine.append(C) 1100 1101 logger.info('need to improve %s channels' % len(to_refine)) 1102 return goal_lum, to_refine
1103
1104 - def update_html(self):
1105 """update the html from this object since it contains all the information""" 1106 1107 1108 run = self.cmd.results.current['run_name'] 1109 if not os.path.exists(pjoin(self.cmd.me_dir, 'HTML', run)): 1110 os.mkdir(pjoin(self.cmd.me_dir, 'HTML', run)) 1111 1112 unit = self.cmd.results.unit 1113 P_text = "" 1114 if self.results: 1115 Presults = self.results 1116 else: 1117 self.results = sum_html.collect_result(self.cmd, None) 1118 Presults = self.results 1119 1120 for P_comb in Presults: 1121 P_text += P_comb.get_html(run, unit, self.cmd.me_dir) 1122 1123 Presults.write_results_dat(pjoin(self.cmd.me_dir,'SubProcesses', 'results.dat')) 1124 1125 fsock = open(pjoin(self.cmd.me_dir, 'HTML', run, 'results.html'),'w') 1126 fsock.write(sum_html.results_header) 1127 fsock.write('%s <dl>' % Presults.get_html(run, unit, self.cmd.me_dir)) 1128 fsock.write('%s </dl></body>' % P_text) 1129 1130 self.cmd.results.add_detail('cross', Presults.xsec) 1131 self.cmd.results.add_detail('error', Presults.xerru) 1132 1133 return Presults.xsec, Presults.xerru
1134
1135 1136 -class gen_ximprove_v4(gen_ximprove):
1137 1138 # some hardcoded value which impact the generation 1139 gen_events_security = 1.2 # multiply the number of requested event by this number for security 1140 combining_job = 0 # allow to run multiple channel in sequence 1141 max_request_event = 1000 # split jobs if a channel if it needs more than that 1142 max_event_in_iter = 5000 1143 min_event_in_iter = 1000 1144 max_splitting = 130 # maximum duplication of a given channel 1145 min_iter = 3 1146 max_iter = 9 1147 keep_grid_for_refine = False # only apply if needed to split the job 1148 1149 1150
1151 - def __init__(self, cmd, opt=None):
1152 1153 super(gen_ximprove_v4, self).__init__(cmd, opt) 1154 1155 if cmd.opts['accuracy'] < cmd._survey_options['accuracy'][1]: 1156 self.increase_precision(cmd._survey_options['accuracy'][1]/cmd.opts['accuracy'])
1157
1158 - def reset_multijob(self):
1159 1160 for path in misc.glob(pjoin('*', '*','multijob.dat'), pjoin(self.me_dir, 'SubProcesses')): 1161 open(path,'w').write('0\n')
1162
1163 - def write_multijob(self, Channel, nb_split):
1164 """ """ 1165 if nb_split <=1: 1166 return 1167 f = open(pjoin(self.me_dir, 'SubProcesses', Channel.get('name'), 'multijob.dat'), 'w') 1168 f.write('%i\n' % nb_split) 1169 f.close()
1170
1171 - def increase_precision(self, rate=3):
1172 #misc.sprint(rate) 1173 if rate < 3: 1174 self.max_event_in_iter = 20000 1175 self.min_events = 7500 1176 self.gen_events_security = 1.3 1177 else: 1178 rate = rate -2 1179 self.max_event_in_iter = int((rate+1) * 10000) 1180 self.min_events = int(rate+2) * 2500 1181 self.gen_events_security = 1 + 0.1 * (rate+2) 1182 1183 if int(self.nhel) == 1: 1184 self.min_event_in_iter *= 2**(self.cmd.proc_characteristics['nexternal']//3) 1185 self.max_event_in_iter *= 2**(self.cmd.proc_characteristics['nexternal']//2)
1186 1187 1188 1189 alphabet = "abcdefghijklmnopqrstuvwxyz"
1190 - def get_job_for_event(self):
1191 """generate the script in order to generate a given number of event""" 1192 # correspond to write_gen in the fortran version 1193 1194 1195 goal_lum, to_refine = self.find_job_for_event() 1196 1197 #reset the potential multijob of previous run 1198 self.reset_multijob() 1199 1200 jobs = [] # list of the refine if some job are split is list of 1201 # dict with the parameter of the run. 1202 1203 # try to have a smart load on the cluster (not really important actually) 1204 if self.combining_job >1: 1205 # add a nice ordering for the jobs 1206 new_order = [] 1207 if self.combining_job % 2 == 0: 1208 for i in range(len(to_refine) //2): 1209 new_order.append(to_refine[i]) 1210 new_order.append(to_refine[-i-1]) 1211 if len(to_refine) % 2: 1212 new_order.append(to_refine[i+1]) 1213 else: 1214 for i in range(len(to_refine) //3): 1215 new_order.append(to_refine[i]) 1216 new_order.append(to_refine[-2*i-1]) 1217 new_order.append(to_refine[-2*i-2]) 1218 if len(to_refine) % 3 == 1: 1219 new_order.append(to_refine[i+1]) 1220 elif len(to_refine) % 3 == 2: 1221 new_order.append(to_refine[i+2]) 1222 #ensure that the reordering is done nicely 1223 assert set([id(C) for C in to_refine]) == set([id(C) for C in new_order]) 1224 to_refine = new_order 1225 1226 1227 # loop over the channel to refine 1228 for C in to_refine: 1229 #1. Compute the number of points are needed to reach target 1230 needed_event = goal_lum*C.get('axsec') 1231 nb_split = int(max(1,((needed_event-1)// self.max_request_event) +1)) 1232 if not self.split_channels: 1233 nb_split = 1 1234 if nb_split > self.max_splitting: 1235 nb_split = self.max_splitting 1236 nb_split=max(1, nb_split) 1237 1238 1239 #2. estimate how many points we need in each iteration 1240 if C.get('nunwgt') > 0: 1241 nevents = needed_event / nb_split * (C.get('nevents') / C.get('nunwgt')) 1242 #split by iter 1243 nevents = int(nevents / (2**self.min_iter-1)) 1244 else: 1245 nevents = self.max_event_in_iter 1246 1247 if nevents < self.min_event_in_iter: 1248 nb_split = int(nb_split * nevents / self.min_event_in_iter) + 1 1249 nevents = self.min_event_in_iter 1250 # 1251 # forbid too low/too large value 1252 nevents = max(self.min_event_in_iter, min(self.max_event_in_iter, nevents)) 1253 logger.debug("%s : need %s event. Need %s split job of %s points", C.name, needed_event, nb_split, nevents) 1254 1255 1256 # write the multi-job information 1257 self.write_multijob(C, nb_split) 1258 1259 packet = cluster.Packet((C.parent_name, C.name), 1260 combine_runs.CombineRuns, 1261 (pjoin(self.me_dir, 'SubProcesses', C.parent_name)), 1262 {"subproc": C.name, "nb_split":nb_split}) 1263 1264 1265 #create the info dict assume no splitting for the default 1266 info = {'name': self.cmd.results.current['run_name'], 1267 'script_name': 'unknown', 1268 'directory': C.name, # need to be change for splitted job 1269 'P_dir': C.parent_name, 1270 'Ppath': pjoin(self.cmd.me_dir, 'SubProcesses', C.parent_name), 1271 'offset': 1, # need to be change for splitted job 1272 'nevents': nevents, 1273 'maxiter': self.max_iter, 1274 'miniter': self.min_iter, 1275 'precision': -goal_lum/nb_split, 1276 'nhel': self.run_card['nhel'], 1277 'channel': C.name.replace('G',''), 1278 'grid_refinment' : 0, #no refinment of the grid 1279 'base_directory': '', #should be change in splitted job if want to keep the grid 1280 'packet': packet, 1281 } 1282 1283 if nb_split == 1: 1284 jobs.append(info) 1285 else: 1286 for i in range(nb_split): 1287 new_info = dict(info) 1288 new_info['offset'] = i+1 1289 new_info['directory'] += self.alphabet[i % 26] + str((i+1)//26) 1290 if self.keep_grid_for_refine: 1291 new_info['base_directory'] = info['directory'] 1292 jobs.append(new_info) 1293 1294 self.create_ajob(pjoin(self.me_dir, 'SubProcesses', 'refine.sh'), jobs)
1295 1296
1297 - def create_ajob(self, template, jobs, write_dir=None):
1298 """create the ajob""" 1299 1300 if not jobs: 1301 return 1302 1303 if not write_dir: 1304 write_dir = pjoin(self.me_dir, 'SubProcesses') 1305 1306 #filter the job according to their SubProcess directory # no mix submition 1307 P2job= collections.defaultdict(list) 1308 for j in jobs: 1309 P2job[j['P_dir']].append(j) 1310 if len(P2job) >1: 1311 for P in P2job.values(): 1312 self.create_ajob(template, P, write_dir) 1313 return 1314 1315 1316 #Here we can assume that all job are for the same directory. 1317 path = pjoin(write_dir, jobs[0]['P_dir']) 1318 1319 template_text = open(template, 'r').read() 1320 # special treatment if needed to combine the script 1321 # computes how many submition miss one job 1322 if self.combining_job > 1: 1323 skip1=0 1324 n_channels = len(jobs) 1325 nb_sub = n_channels // self.combining_job 1326 nb_job_in_last = n_channels % self.combining_job 1327 if nb_sub == 0: 1328 nb_sub = 1 1329 nb_job_in_last =0 1330 if nb_job_in_last: 1331 nb_sub +=1 1332 skip1 = self.combining_job - nb_job_in_last 1333 if skip1 > nb_sub: 1334 self.combining_job -=1 1335 return self.create_ajob(template, jobs, write_dir) 1336 combining_job = self.combining_job 1337 else: 1338 #define the variable for combining jobs even in not combine mode 1339 #such that we can use the same routine 1340 skip1=0 1341 combining_job =1 1342 nb_sub = len(jobs) 1343 1344 1345 nb_use = 0 1346 for i in range(nb_sub): 1347 script_number = i+1 1348 if i < skip1: 1349 nb_job = combining_job -1 1350 else: 1351 nb_job = min(combining_job, len(jobs)) 1352 fsock = open(pjoin(path, 'ajob%i' % script_number), 'w') 1353 for j in range(nb_use, nb_use + nb_job): 1354 if j> len(jobs): 1355 break 1356 info = jobs[j] 1357 info['script_name'] = 'ajob%i' % script_number 1358 info['keeplog'] = 'false' 1359 if "base_directory" not in info: 1360 info["base_directory"] = "./" 1361 fsock.write(template_text % info) 1362 nb_use += nb_job 1363 1364 fsock.close() 1365 return script_number
1366
1367 - def get_job_for_precision(self):
1368 """create the ajob to achieve a give precision on the total cross-section""" 1369 1370 1371 assert self.err_goal <=1 1372 xtot = abs(self.results.xsec) 1373 logger.info("Working on precision: %s %%" %(100*self.err_goal)) 1374 all_channels = sum([list(P) for P in self.results if P.mfactor],[]) 1375 limit = self.err_goal * xtot / len(all_channels) 1376 to_refine = [] 1377 rerr = 0 #error of the job not directly selected 1378 for C in all_channels: 1379 cerr = C.mfactor*(C.xerru + len(all_channels)*C.xerrc) 1380 if cerr > abs(limit): 1381 to_refine.append(C) 1382 else: 1383 rerr += cerr 1384 rerr *=rerr 1385 if not len(to_refine): 1386 return 1387 1388 # change limit since most don't contribute 1389 limit = math.sqrt((self.err_goal * xtot)**2 - rerr/math.sqrt(len(to_refine))) 1390 for C in to_refine[:]: 1391 cerr = C.mfactor*(C.xerru + len(to_refine)*C.xerrc) 1392 if cerr < limit: 1393 to_refine.remove(C) 1394 1395 # all the channel are now selected. create the channel information 1396 logger.info('need to improve %s channels' % len(to_refine)) 1397 1398 1399 jobs = [] # list of the refine if some job are split is list of 1400 # dict with the parameter of the run. 1401 1402 # loop over the channel to refine 1403 for C in to_refine: 1404 1405 #1. Determine how many events we need in each iteration 1406 yerr = C.mfactor*(C.xerru+len(to_refine)*C.xerrc) 1407 nevents = 0.2*C.nevents*(yerr/limit)**2 1408 1409 nb_split = int((nevents*(C.nunwgt/C.nevents)/self.max_request_event/ (2**self.min_iter-1))**(2/3)) 1410 nb_split = max(nb_split, 1) 1411 # **(2/3) to slow down the increase in number of jobs 1412 if nb_split > self.max_splitting: 1413 nb_split = self.max_splitting 1414 1415 if nb_split >1: 1416 nevents = nevents / nb_split 1417 self.write_multijob(C, nb_split) 1418 # forbid too low/too large value 1419 nevents = min(self.min_event_in_iter, max(self.max_event_in_iter, nevents)) 1420 1421 1422 #create the info dict assume no splitting for the default 1423 info = {'name': self.cmd.results.current['run_name'], 1424 'script_name': 'unknown', 1425 'directory': C.name, # need to be change for splitted job 1426 'P_dir': C.parent_name, 1427 'Ppath': pjoin(self.cmd.me_dir, 'SubProcesses', C.parent_name), 1428 'offset': 1, # need to be change for splitted job 1429 'nevents': nevents, 1430 'maxiter': self.max_iter, 1431 'miniter': self.min_iter, 1432 'precision': yerr/math.sqrt(nb_split)/(C.get('xsec')+ yerr), 1433 'nhel': self.run_card['nhel'], 1434 'channel': C.name.replace('G',''), 1435 'grid_refinment' : 1 1436 } 1437 1438 if nb_split == 1: 1439 jobs.append(info) 1440 else: 1441 for i in range(nb_split): 1442 new_info = dict(info) 1443 new_info['offset'] = i+1 1444 new_info['directory'] += self.alphabet[i % 26] + str((i+1)//26) 1445 jobs.append(new_info) 1446 self.create_ajob(pjoin(self.me_dir, 'SubProcesses', 'refine.sh'), jobs)
1447
1448 - def update_html(self):
1449 """update the html from this object since it contains all the information""" 1450 1451 1452 run = self.cmd.results.current['run_name'] 1453 if not os.path.exists(pjoin(self.cmd.me_dir, 'HTML', run)): 1454 os.mkdir(pjoin(self.cmd.me_dir, 'HTML', run)) 1455 1456 unit = self.cmd.results.unit 1457 P_text = "" 1458 if self.results: 1459 Presults = self.results 1460 else: 1461 self.results = sum_html.collect_result(self.cmd, None) 1462 Presults = self.results 1463 1464 for P_comb in Presults: 1465 P_text += P_comb.get_html(run, unit, self.cmd.me_dir) 1466 1467 Presults.write_results_dat(pjoin(self.cmd.me_dir,'SubProcesses', 'results.dat')) 1468 1469 fsock = open(pjoin(self.cmd.me_dir, 'HTML', run, 'results.html'),'w') 1470 fsock.write(sum_html.results_header) 1471 fsock.write('%s <dl>' % Presults.get_html(run, unit, self.cmd.me_dir)) 1472 fsock.write('%s </dl></body>' % P_text) 1473 1474 self.cmd.results.add_detail('cross', Presults.xsec) 1475 self.cmd.results.add_detail('error', Presults.xerru) 1476 1477 return Presults.xsec, Presults.xerru
1478
1479 1480 1481 1482 -class gen_ximprove_v4_nogridupdate(gen_ximprove_v4):
1483 1484 # some hardcoded value which impact the generation 1485 gen_events_security = 1.1 # multiply the number of requested event by this number for security 1486 combining_job = 0 # allow to run multiple channel in sequence 1487 max_request_event = 400 # split jobs if a channel if it needs more than that 1488 max_event_in_iter = 500 1489 min_event_in_iter = 250 1490 max_splitting = 260 # maximum duplication of a given channel 1491 min_iter = 2 1492 max_iter = 6 1493 keep_grid_for_refine = True 1494 1495
1496 - def __init__(self, cmd, opt=None):
1497 1498 gen_ximprove.__init__(cmd, opt) 1499 1500 if cmd.proc_characteristics['loopinduced'] and \ 1501 cmd.proc_characteristics['nexternal'] > 2: 1502 self.increase_parralelization(cmd.proc_characteristics['nexternal'])
1503
1504 - def increase_parralelization(self, nexternal):
1505 1506 self.max_splitting = 1000 1507 1508 if self.run_card['refine_evt_by_job'] != -1: 1509 pass 1510 elif nexternal == 3: 1511 self.max_request_event = 200 1512 elif nexternal == 4: 1513 self.max_request_event = 100 1514 elif nexternal >= 5: 1515 self.max_request_event = 50 1516 self.min_event_in_iter = 125 1517 self.max_iter = 5
1518
1519 -class gen_ximprove_share(gen_ximprove, gensym):
1520 """Doing the refine in multicore. Each core handle a couple of PS point.""" 1521 1522 nb_ps_by_job = 2000 1523 mode = "refine" 1524 gen_events_security = 1.15 1525 # Note the real security is lower since we stop the jobs if they are at 96% 1526 # of this target. 1527
1528 - def __init__(self, *args, **opts):
1529 1530 super(gen_ximprove_share, self).__init__(*args, **opts) 1531 self.generated_events = {} 1532 self.splitted_for_dir = lambda x,y : self.splitted_Pdir[(x,y)]
1533 1534
1535 - def get_job_for_event(self):
1536 """generate the script in order to generate a given number of event""" 1537 # correspond to write_gen in the fortran version 1538 1539 1540 goal_lum, to_refine = self.find_job_for_event() 1541 self.goal_lum = goal_lum 1542 1543 # loop over the channel to refine to find the number of PS point to launch 1544 total_ps_points = 0 1545 channel_to_ps_point = [] 1546 for C in to_refine: 1547 #0. remove previous events files 1548 try: 1549 os.remove(pjoin(self.me_dir, "SubProcesses",C.parent_name, C.name, "events.lhe")) 1550 except: 1551 pass 1552 1553 #1. Compute the number of points are needed to reach target 1554 needed_event = goal_lum*C.get('axsec') 1555 if needed_event == 0: 1556 continue 1557 #2. estimate how many points we need in each iteration 1558 if C.get('nunwgt') > 0: 1559 nevents = needed_event * (C.get('nevents') / C.get('nunwgt')) 1560 #split by iter 1561 nevents = int(nevents / (2**self.min_iter-1)) 1562 else: 1563 nb_split = int(max(1,((needed_event-1)// self.max_request_event) +1)) 1564 if not self.split_channels: 1565 nb_split = 1 1566 if nb_split > self.max_splitting: 1567 nb_split = self.max_splitting 1568 nevents = self.max_event_in_iter * self.max_splitting 1569 else: 1570 nevents = self.max_event_in_iter * nb_split 1571 1572 if nevents > self.max_splitting*self.max_event_in_iter: 1573 logger.warning("Channel %s/%s has a very low efficiency of unweighting. Might not be possible to reach target" % \ 1574 (C.name, C.parent_name)) 1575 nevents = self.max_event_in_iter * self.max_splitting 1576 1577 total_ps_points += nevents 1578 channel_to_ps_point.append((C, nevents)) 1579 1580 if self.cmd.options["run_mode"] == 1: 1581 if self.cmd.options["cluster_size"]: 1582 nb_ps_by_job = total_ps_points /int(self.cmd.options["cluster_size"]) 1583 else: 1584 nb_ps_by_job = self.nb_ps_by_job 1585 elif self.cmd.options["run_mode"] == 2: 1586 remain = total_ps_points % self.cmd.options["nb_core"] 1587 if remain: 1588 nb_ps_by_job = 1 + (total_ps_points - remain) / self.cmd.options["nb_core"] 1589 else: 1590 nb_ps_by_job = total_ps_points / self.cmd.options["nb_core"] 1591 else: 1592 nb_ps_by_job = self.nb_ps_by_job 1593 1594 nb_ps_by_job = int(max(nb_ps_by_job, 500)) 1595 1596 for C, nevents in channel_to_ps_point: 1597 if nevents % nb_ps_by_job: 1598 nb_job = 1 + int(nevents // nb_ps_by_job) 1599 else: 1600 nb_job = int(nevents // nb_ps_by_job) 1601 submit_ps = min(nevents, nb_ps_by_job) 1602 if nb_job == 1: 1603 submit_ps = max(submit_ps, self.min_event_in_iter) 1604 self.create_resubmit_one_iter(C.parent_name, C.name[1:], submit_ps, nb_job, step=0) 1605 needed_event = goal_lum*C.get('xsec') 1606 logger.debug("%s/%s : need %s event. Need %s split job of %s points", C.parent_name, C.name, needed_event, nb_job, submit_ps)
1607 1608
1609 - def combine_iteration(self, Pdir, G, step):
1610 1611 grid_calculator, cross, error = self.combine_grid(Pdir, G, step) 1612 1613 # collect all the generated_event 1614 Gdirs = [] #build the the list of directory 1615 for i in range(self.splitted_for_dir(Pdir, G)): 1616 path = pjoin(Pdir, "G%s_%s" % (G, i+1)) 1617 Gdirs.append(path) 1618 assert len(grid_calculator.results) == len(Gdirs) == self.splitted_for_dir(Pdir, G) 1619 1620 1621 # Check how many events are going to be kept after un-weighting. 1622 needed_event = cross * self.goal_lum 1623 if needed_event == 0: 1624 return 0 1625 # check that the number of events requested is not higher than the actual 1626 # total number of events to generate. 1627 if self.err_goal >=1: 1628 if needed_event > self.gen_events_security * self.err_goal: 1629 needed_event = int(self.gen_events_security * self.err_goal) 1630 1631 if (Pdir, G) in self.generated_events: 1632 old_nunwgt, old_maxwgt = self.generated_events[(Pdir, G)] 1633 else: 1634 old_nunwgt, old_maxwgt = 0, 0 1635 1636 if old_nunwgt == 0 and os.path.exists(pjoin(Pdir,"G%s" % G, "events.lhe")): 1637 # possible for second refine. 1638 lhe = lhe_parser.EventFile(pjoin(Pdir,"G%s" % G, "events.lhe")) 1639 old_nunwgt = lhe.unweight(None, trunc_error=0.005, log_level=0) 1640 old_maxwgt = lhe.max_wgt 1641 1642 1643 1644 maxwgt = max(grid_calculator.get_max_wgt(), old_maxwgt) 1645 new_evt = grid_calculator.get_nunwgt(maxwgt) 1646 efficiency = new_evt / sum([R.nevents for R in grid_calculator.results]) 1647 nunwgt = old_nunwgt * old_maxwgt / maxwgt 1648 nunwgt += new_evt 1649 1650 # check the number of event for this iteration alone 1651 one_iter_nb_event = max(grid_calculator.get_nunwgt(),1) 1652 drop_previous_iteration = False 1653 # compare the number of events to generate if we discard the previous iteration 1654 n_target_one_iter = (needed_event-one_iter_nb_event) / ( one_iter_nb_event/ sum([R.nevents for R in grid_calculator.results])) 1655 n_target_combined = (needed_event-nunwgt) / efficiency 1656 if n_target_one_iter < n_target_combined: 1657 # the last iteration alone has more event that the combine iteration. 1658 # it is therefore interesting to drop previous iteration. 1659 drop_previous_iteration = True 1660 nunwgt = one_iter_nb_event 1661 maxwgt = grid_calculator.get_max_wgt() 1662 new_evt = nunwgt 1663 efficiency = ( one_iter_nb_event/ sum([R.nevents for R in grid_calculator.results])) 1664 1665 try: 1666 if drop_previous_iteration: 1667 raise IOError 1668 output_file = open(pjoin(Pdir,"G%s" % G, "events.lhe"), 'a') 1669 except IOError: 1670 output_file = open(pjoin(Pdir,"G%s" % G, "events.lhe"), 'w') 1671 1672 misc.call(["cat"] + [pjoin(d, "events.lhe") for d in Gdirs], 1673 stdout=output_file) 1674 output_file.close() 1675 # For large number of iteration. check the number of event by doing the 1676 # real unweighting. 1677 if nunwgt < 0.6 * needed_event and step > self.min_iter: 1678 lhe = lhe_parser.EventFile(output_file.name) 1679 old_nunwgt =nunwgt 1680 nunwgt = lhe.unweight(None, trunc_error=0.01, log_level=0) 1681 1682 1683 self.generated_events[(Pdir, G)] = (nunwgt, maxwgt) 1684 1685 # misc.sprint("Adding %s event to %s. Currently at %s" % (new_evt, G, nunwgt)) 1686 # check what to do 1687 if nunwgt >= int(0.96*needed_event)+1: # 0.96*1.15=1.10 =real security 1688 # We did it. 1689 logger.info("found enough event for %s/G%s" % (os.path.basename(Pdir), G)) 1690 self.write_results(grid_calculator, cross, error, Pdir, G, step, efficiency) 1691 return 0 1692 elif step >= self.max_iter: 1693 logger.debug("fail to find enough event") 1694 self.write_results(grid_calculator, cross, error, Pdir, G, step, efficiency) 1695 return 0 1696 1697 nb_split_before = len(grid_calculator.results) 1698 nevents = grid_calculator.results[0].nevents 1699 if nevents == 0: # possible if some integral returns 0 1700 nevents = max(g.nevents for g in grid_calculator.results) 1701 1702 need_ps_point = (needed_event - nunwgt)/(efficiency+1e-99) 1703 need_job = need_ps_point // nevents + 1 1704 1705 if step < self.min_iter: 1706 # This is normal but check if we are on the good track 1707 job_at_first_iter = nb_split_before/2**(step-1) 1708 expected_total_job = job_at_first_iter * (2**self.min_iter-1) 1709 done_job = job_at_first_iter * (2**step-1) 1710 expected_remaining_job = expected_total_job - done_job 1711 1712 logger.debug("efficiency status (smaller is better): %s", need_job/expected_remaining_job) 1713 # increase if needed but not too much 1714 need_job = min(need_job, expected_remaining_job*1.25) 1715 1716 nb_job = (need_job-0.5)//(2**(self.min_iter-step)-1) + 1 1717 nb_job = max(1, nb_job) 1718 grid_calculator.write_grid_for_submission(Pdir,G, 1719 self.splitted_for_dir(Pdir, G), nb_job*nevents ,mode=self.mode, 1720 conservative_factor=self.max_iter) 1721 logger.info("%s/G%s is at %i/%i (%.2g%%) event. Resubmit %i job at iteration %i." \ 1722 % (os.path.basename(Pdir), G, int(nunwgt),int(needed_event)+1, 1723 (float(nunwgt)/needed_event)*100.0 if needed_event>0.0 else 0.0, 1724 nb_job, step)) 1725 self.create_resubmit_one_iter(Pdir, G, nevents, nb_job, step) 1726 #self.create_job(Pdir, G, nb_job, nevents, step) 1727 1728 elif step < self.max_iter: 1729 if step + 1 == self.max_iter: 1730 need_job = 1.20 * need_job # avoid to have just too few event. 1731 1732 nb_job = int(min(need_job, nb_split_before*1.5)) 1733 grid_calculator.write_grid_for_submission(Pdir,G, 1734 self.splitted_for_dir(Pdir, G), nb_job*nevents ,mode=self.mode, 1735 conservative_factor=self.max_iter) 1736 1737 1738 logger.info("%s/G%s is at %i/%i ('%.2g%%') event. Resubmit %i job at iteration %i." \ 1739 % (os.path.basename(Pdir), G, int(nunwgt),int(needed_event)+1, 1740 (float(nunwgt)/needed_event)*100.0 if needed_event>0.0 else 0.0, 1741 nb_job, step)) 1742 self.create_resubmit_one_iter(Pdir, G, nevents, nb_job, step) 1743 1744 1745 1746 return 0
1747 1748
1749 - def write_results(self, grid_calculator, cross, error, Pdir, G, step, efficiency):
1750 1751 #compute the value 1752 if cross == 0: 1753 abscross,nw, luminosity = 0, 0, 0 1754 wgt, maxit,nunwgt, wgt, nevents = 0,0,0,0,0 1755 error = 0 1756 else: 1757 grid_calculator.results.compute_values() 1758 abscross = self.abscross[(Pdir,G)]/self.sigma[(Pdir,G)] 1759 nunwgt, wgt = self.generated_events[(Pdir, G)] 1760 nw = int(nunwgt / efficiency) 1761 nunwgt = int(nunwgt) 1762 maxit = step 1763 nevents = nunwgt 1764 # make the unweighting to compute the number of events: 1765 luminosity = nunwgt/cross 1766 1767 #format the results.dat 1768 def fstr(nb): 1769 data = '%E' % nb 1770 nb, power = data.split('E') 1771 nb = float(nb) /10 1772 power = int(power) + 1 1773 return '%.5fE%+03i' %(nb,power)
1774 line = '%s %s %s %i %i %i %i %s %s %s 0.0 0.0 0\n' % \ 1775 (fstr(cross), fstr(error*cross), fstr(error*cross), 1776 nevents, nw, maxit,nunwgt, 1777 fstr(luminosity), fstr(wgt), fstr(abscross)) 1778 1779 fsock = open(pjoin(self.me_dir,'SubProcesses' , Pdir, 'G%s' % G, 1780 'results.dat'),'w') 1781 fsock.writelines(line) 1782 fsock.close()
1783
1784 1785 1786 1787 -class gen_ximprove_gridpack(gen_ximprove_v4):
1788 1789 min_iter = 1 1790 max_iter = 13 1791 max_request_event = 1e12 # split jobs if a channel if it needs more than that 1792 max_event_in_iter = 4000 1793 min_event_in_iter = 500 1794 combining_job = sys.maxsize 1795 gen_events_security = 1.00 1796
1797 - def __new__(cls, *args, **opts):
1798 1799 return super(gen_ximprove, cls).__new__(cls, *args, **opts)
1800
1801 - def __init__(self, *args, **opts):
1802 1803 self.ngran = -1 1804 self.gscalefact = {} 1805 self.readonly = False 1806 if 'ngran' in opts: 1807 self.gran = opts['ngran'] 1808 # del opts['ngran'] 1809 if 'readonly' in opts: 1810 self.readonly = opts['readonly'] 1811 super(gen_ximprove_gridpack,self).__init__(*args, **opts) 1812 if self.ngran == -1: 1813 self.ngran = 1
1814
1815 - def find_job_for_event(self):
1816 """return the list of channel that need to be improved""" 1817 import random 1818 1819 assert self.err_goal >=1 1820 self.err_goal = int(self.err_goal) 1821 self.gscalefact = {} 1822 1823 xtot = self.results.axsec 1824 goal_lum = self.err_goal/(xtot+1e-99) #pb^-1 1825 # logger.info('Effective Luminosity %s pb^-1', goal_lum) 1826 1827 all_channels = sum([list(P) for P in self.results],[]) 1828 all_channels.sort(cmp= lambda x,y: 1 if y.get('luminosity') - \ 1829 x.get('luminosity') > 0 else -1) 1830 1831 to_refine = [] 1832 for C in all_channels: 1833 tag = C.get('name') 1834 self.gscalefact[tag] = 0 1835 R = random.random() 1836 if C.get('axsec') == 0: 1837 continue 1838 if (goal_lum * C.get('axsec') < R*self.ngran ): 1839 continue # no event to generate events 1840 self.gscalefact[tag] = max(1, 1/(goal_lum * C.get('axsec')/ self.ngran)) 1841 #need to generate events 1842 logger.debug('request events for ', C.get('name'), 'cross=', 1843 C.get('axsec'), 'needed events = ', goal_lum * C.get('axsec')) 1844 to_refine.append(C) 1845 1846 logger.info('need to improve %s channels' % len(to_refine)) 1847 return goal_lum, to_refine
1848
1849 - def get_job_for_event(self):
1850 """generate the script in order to generate a given number of event""" 1851 # correspond to write_gen in the fortran version 1852 1853 1854 goal_lum, to_refine = self.find_job_for_event() 1855 1856 jobs = [] # list of the refine if some job are split is list of 1857 # dict with the parameter of the run. 1858 1859 # loop over the channel to refine 1860 for C in to_refine: 1861 #1. Compute the number of points are needed to reach target 1862 needed_event = max(goal_lum*C.get('axsec'), self.ngran) 1863 nb_split = 1 1864 1865 #2. estimate how many points we need in each iteration 1866 if C.get('nunwgt') > 0: 1867 nevents = needed_event / nb_split * (C.get('nevents') / C.get('nunwgt')) 1868 #split by iter 1869 nevents = int(nevents / (2**self.min_iter-1)) 1870 else: 1871 nevents = self.max_event_in_iter 1872 1873 if nevents < self.min_event_in_iter: 1874 nevents = self.min_event_in_iter 1875 # 1876 # forbid too low/too large value 1877 nevents = max(self.min_event_in_iter, min(self.max_event_in_iter, nevents)) 1878 logger.debug("%s : need %s event. Need %s split job of %s points", C.name, needed_event, nb_split, nevents) 1879 1880 1881 #create the info dict assume no splitting for the default 1882 info = {'name': self.cmd.results.current['run_name'], 1883 'script_name': 'unknown', 1884 'directory': C.name, # need to be change for splitted job 1885 'P_dir': os.path.basename(C.parent_name), 1886 'offset': 1, # need to be change for splitted job 1887 'Ppath': pjoin(self.cmd.me_dir, 'SubProcesses', C.parent_name), 1888 'nevents': nevents, #int(nevents*self.gen_events_security)+1, 1889 'maxiter': self.max_iter, 1890 'miniter': self.min_iter, 1891 'precision': -1*int(needed_event)/C.get('axsec'), 1892 'requested_event': needed_event, 1893 'nhel': self.run_card['nhel'], 1894 'channel': C.name.replace('G',''), 1895 'grid_refinment' : 0, #no refinment of the grid 1896 'base_directory': '', #should be change in splitted job if want to keep the grid 1897 'packet': None, 1898 } 1899 1900 1901 jobs.append(info) 1902 1903 1904 write_dir = '.' if self.readonly else None 1905 self.create_ajob(pjoin(self.me_dir, 'SubProcesses', 'refine.sh'), jobs, write_dir) 1906 1907 done = [] 1908 for j in jobs: 1909 if j['P_dir'] in done: 1910 continue 1911 done.append(j['P_dir']) 1912 # set the working directory path. 1913 pwd = pjoin(os.getcwd(),j['P_dir']) if self.readonly else pjoin(self.me_dir, 'SubProcesses', j['P_dir']) 1914 exe = pjoin(pwd, 'ajob1') 1915 st = os.stat(exe) 1916 os.chmod(exe, st.st_mode | stat.S_IEXEC) 1917 1918 # run the code\ 1919 cluster.onecore.launch_and_wait(exe, cwd=pwd, packet_member=j['packet']) 1920 write_dir = '.' if self.readonly else pjoin(self.me_dir, 'SubProcesses') 1921 1922 self.check_events(goal_lum, to_refine, jobs, write_dir)
1923
1924 - def check_events(self, goal_lum, to_refine, jobs, Sdir):
1925 """check that we get the number of requested events if not resubmit.""" 1926 1927 new_jobs = [] 1928 1929 for C, job_info in zip(to_refine, jobs): 1930 P = job_info['P_dir'] 1931 G = job_info['channel'] 1932 axsec = C.get('axsec') 1933 requested_events= job_info['requested_event'] 1934 1935 1936 new_results = sum_html.OneResult((P,G)) 1937 new_results.read_results(pjoin(Sdir,P, 'G%s'%G, 'results.dat')) 1938 1939 # need to resubmit? 1940 if new_results.get('nunwgt') < requested_events: 1941 pwd = pjoin(os.getcwd(),job_info['P_dir'],'G%s'%G) if self.readonly else \ 1942 pjoin(self.me_dir, 'SubProcesses', job_info['P_dir'],'G%s'%G) 1943 job_info['requested_event'] -= new_results.get('nunwgt') 1944 job_info['precision'] -= -1*job_info['requested_event']/axsec 1945 job_info['offset'] += 1 1946 new_jobs.append(job_info) 1947 files.mv(pjoin(pwd, 'events.lhe'), pjoin(pwd, 'events.lhe.previous')) 1948 1949 if new_jobs: 1950 self.create_ajob(pjoin(self.me_dir, 'SubProcesses', 'refine.sh'), new_jobs, Sdir) 1951 1952 done = [] 1953 for j in new_jobs: 1954 if j['P_dir'] in done: 1955 continue 1956 G = j['channel'] 1957 # set the working directory path. 1958 pwd = pjoin(os.getcwd(),j['P_dir']) if self.readonly \ 1959 else pjoin(self.me_dir, 'SubProcesses', j['P_dir']) 1960 exe = pjoin(pwd, 'ajob1') 1961 st = os.stat(exe) 1962 os.chmod(exe, st.st_mode | stat.S_IEXEC) 1963 1964 # run the code 1965 cluster.onecore.launch_and_wait(exe, cwd=pwd, packet_member=j['packet']) 1966 pwd = pjoin(pwd, 'G%s'%G) 1967 # concatanate with old events file 1968 files.put_at_end(pjoin(pwd, 'events.lhe'),pjoin(pwd, 'events.lhe.previous')) 1969 1970 return self.check_events(goal_lum, to_refine, new_jobs, Sdir)
1971