1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """ Command interface for MadSpin """
16 from __future__ import division
17 import difflib
18 import logging
19 import math
20 import os
21 import re
22 import shutil
23 import tempfile
24 import time
25 import subprocess
26 from subprocess import Popen, PIPE, STDOUT
27
28
29 pjoin = os.path.join
30
31 import madgraph.interface.extended_cmd as extended_cmd
32 import madgraph.interface.madgraph_interface as mg_interface
33 import madgraph.interface.master_interface as master_interface
34 import madgraph.interface.common_run_interface as common_run_interface
35 import madgraph.iolibs.files as files
36 import MadSpin.interface_madspin as madspin_interface
37 import madgraph.various.misc as misc
38 import madgraph.various.banner as banner
39 import madgraph.various.lhe_parser as lhe_parser
40 import madgraph.various.combine_plots as combine_plots
41
42 import models.import_ufo as import_ufo
43 import models.check_param_card as check_param_card
44 import MadSpin.decay as madspin
45
46
47 logger = logging.getLogger('decay.stdout')
48 logger_stderr = logging.getLogger('decay.stderr')
49 cmd_logger = logging.getLogger('cmdprint2')
54 """Basic interface for reweighting operation"""
55
56 prompt = 'Reweight>'
57 debug_output = 'Reweight_debug'
58
59 @misc.mute_logger()
60 - def __init__(self, event_path=None, allow_madspin=False, *completekey, **stdin):
61 """initialize the interface with potentially an event_path"""
62
63 if not event_path:
64 cmd_logger.info('************************************************************')
65 cmd_logger.info('* *')
66 cmd_logger.info('* Welcome to Reweight Module *')
67 cmd_logger.info('* *')
68 cmd_logger.info('************************************************************')
69 extended_cmd.Cmd.__init__(self, *completekey, **stdin)
70
71 self.model = None
72 self.has_standalone_dir = False
73
74 self.options = {'curr_dir': os.path.realpath(os.getcwd())}
75
76 self.events_file = None
77 self.processes = {}
78 self.mg5cmd = master_interface.MasterCmd()
79 self.seed = None
80
81 if event_path:
82 logger.info("Extracting the banner ...")
83 self.do_import(event_path, allow_madspin=allow_madspin)
84
85
86 self.calculator = {}
87 self.calculator_nbcall = {}
88
89
90 self.all_cross_section = {}
91
92 - def do_import(self, inputfile, allow_madspin=False):
93 """import the event file"""
94
95
96 self.options['curr_dir'] = os.path.realpath(os.path.dirname(inputfile))
97 if os.path.basename(os.path.dirname(os.path.dirname(inputfile))) == 'Events':
98 self.options['curr_dir'] = pjoin(self.options['curr_dir'],
99 os.path.pardir, os.pardir)
100
101
102 if not os.path.exists(inputfile):
103 if inputfile.endswith('.gz'):
104 if not os.path.exists(inputfile[:-3]):
105 raise self.InvalidCmd('No such file or directory : %s' % inputfile)
106 else:
107 inputfile = inputfile[:-3]
108 elif os.path.exists(inputfile + '.gz'):
109 inputfile = inputfile + '.gz'
110 else:
111 raise self.InvalidCmd('No such file or directory : %s' % inputfile)
112
113 if inputfile.endswith('.gz'):
114 misc.gunzip(inputfile)
115 inputfile = inputfile[:-3]
116
117
118 self.lhe_input = lhe_parser.EventFile(os.path.realpath(inputfile))
119 if not self.lhe_input.banner:
120 value = self.ask("What is the path to banner", 0, [0], "please enter a path", timeout=0)
121 self.lhe_input.banner = open(value).read()
122 self.banner = self.lhe_input.get_banner()
123
124
125 if 'slha' not in self.banner:
126 misc.sprint(self.banner)
127 self.events_file = None
128 raise self.InvalidCmd('Event file does not contain model information')
129 elif 'mg5proccard' not in self.banner:
130 self.events_file = None
131 raise self.InvalidCmd('Event file does not contain generation information')
132
133 if 'madspin' in self.banner and not allow_madspin:
134 raise self.InvalidCmd('Reweight should be done before running MadSpin')
135
136
137
138 process = self.banner.get_detail('proc_card', 'generate')
139 if '[' in process:
140 msg = 'Reweighting options is valid only for LO events'
141 raise Exception, msg
142 if not process:
143 msg = 'Invalid proc_card information in the file (no generate line):\n %s' % self.banner['mg5proccard']
144 raise Exception, msg
145 process, option = mg_interface.MadGraphCmd.split_process_line(process)
146 self.proc_option = option
147
148 logger.info("process: %s" % process)
149 logger.info("options: %s" % option)
150
151
153 """Check some basic property of the events file"""
154
155 sum_of_weight = 0
156 sum_of_abs_weight = 0
157 negative_event = 0
158 positive_event = 0
159
160 start = time.time()
161 for event_nb,event in enumerate(self.lhe_input):
162
163 if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),10)==0):
164 running_time = misc.format_timer(time.time()-start)
165 logger.info('Event nb %s %s' % (event_nb, running_time))
166 if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
167
168 event.check()
169
170 sum_of_weight += event.wgt
171 sum_of_abs_weight += abs(event.wgt)
172 if event.wgt < 0 :
173 negative_event +=1
174 else:
175 positive_event +=1
176
177 logger.info("total cross-section: %s" % sum_of_weight)
178 logger.info("total abs cross-section: %s" % sum_of_abs_weight)
179 logger.info("fraction of negative event %s", negative_event/(negative_event+positive_event))
180 logger.info("total number of events %s", (negative_event+positive_event))
181 logger.info("negative event %s", negative_event)
182
183
184
185
186 @extended_cmd.debug()
188 "Complete the import command"
189
190 args=self.split_arg(line[0:begidx])
191
192 if len(args) == 1:
193 base_dir = '.'
194 else:
195 base_dir = args[1]
196
197 return self.path_completion(text, base_dir)
198
199
200 if os.path.sep in args[-1] + text:
201 return self.path_completion(text,
202 pjoin(*[a for a in args if \
203 a.endswith(os.path.sep)]))
204
206 """checking the validity of the set command"""
207
208 if len(args) < 2:
209 raise self.InvalidCmd('set command requires at least two argument.')
210
211 valid = ['max_weight','seed','curr_dir']
212 if args[0] not in self.options and args[0] not in valid:
213 raise self.InvalidCmd('Unknown options %s' % args[0])
214
215 if args[0] == 'max_weight':
216 try:
217 args[1] = float(args[1].replace('d','e'))
218 except ValueError:
219 raise self.InvalidCmd('second argument should be a real number.')
220
221 elif args[0] == 'BW_effect':
222 if args[1] in [0, False,'.false.', 'F', 'f', 'False', 'no']:
223 args[1] = 0
224 elif args[1] in [1, True,'.true.', 'T', 't', 'True', 'yes']:
225 args[1] = 1
226 else:
227 raise self.InvalidCmd('second argument should be either T or F.')
228
229 elif args[0] == 'curr_dir':
230 if not os.path.isdir(args[1]):
231 raise self.InvalidCmd('second argument should be a path to a existing directory')
232
233
235 """check the validity of the launch command"""
236
237 if not self.lhe_input:
238 if isinstance(self.lhe_input, lhe_parser.EventFile):
239 self.lhe_input = lhe_parser.EventFile(self.lhe_input.name)
240 else:
241 raise self.InvalidCmd("No events files defined.")
242
244 """help for the launch command"""
245
246 logger.info('''Add to the loaded events a weight associated to a
247 new param_card (to be define). The weight returned is the ratio of the
248 square matrix element by the squared matrix element of production.
249 All scale are kept fix for this re-weighting.''')
250
251
253 """end of the configuration launched the code"""
254
255 args = self.split_arg(line)
256 self.check_launch(args)
257
258 model_line = self.banner.get('proc_card', 'full_model_line')
259
260 if not self.has_standalone_dir:
261 self.create_standalone_directory()
262
263 ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card.dat'), 'w')
264 ff.write(self.banner['slha'])
265 ff.close()
266 ff = open(pjoin(self.me_dir, 'rw_me','Cards', 'param_card_orig.dat'), 'w')
267 ff.write(self.banner['slha'])
268 ff.close()
269 cmd = common_run_interface.CommonRunCmd.ask_edit_card_static(cards=['param_card.dat'],
270 ask=self.ask, pwd=pjoin(self.me_dir,'rw_me'))
271
272 new_card = open(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat')).read()
273
274 if "auto" in new_card.lower():
275 self.mother.check_param_card(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat'))
276 new_card = open(pjoin(self.me_dir, 'rw_me', 'Cards', 'param_card.dat')).read()
277
278
279
280 if 'initrwgt' in self.banner:
281 if 'type=\'mg_reweighting\'' in self.banner['initrwgt']:
282 blockpat = re.compile(r'''<weightgroup type=\'mg_reweighting\'\s*>(?P<text>.*?)</weightgroup>''', re.I+re.M+re.S)
283 before, content, after = blockpat.split(self.banner['initrwgt'])
284 header_rwgt_other = before + after
285 pattern = re.compile('<weight id=\'mg_reweight_(?P<id>\d+)\'>(?P<info>[^<>]*)</weight>', re.S+re.I+re.M)
286 mg_rwgt_info = pattern.findall(content)
287 maxid = 0
288 for i, diff in mg_rwgt_info:
289 if int(i) > maxid:
290 maxid = int(i)
291 maxid += 1
292 rewgtid = maxid
293 else:
294 header_rwgt_other = self.banner['initrwgt']
295 mg_rwgt_info = []
296 rewgtid = 1
297 else:
298 self.banner['initrwgt'] = ''
299 header_rwgt_other = ''
300 mg_rwgt_info = []
301 rewgtid = 1
302
303
304
305
306
307
308 s_orig = self.banner['slha']
309 s_new = new_card
310 old_param = check_param_card.ParamCard(s_orig.splitlines())
311 new_param = check_param_card.ParamCard(s_new.splitlines())
312 card_diff = old_param.create_diff(new_param)
313 if card_diff == '':
314 raise self.InvalidCmd, 'original card and new card are identical'
315 mg_rwgt_info.append((str(rewgtid), card_diff))
316
317
318 self.banner['initrwgt'] = header_rwgt_other
319 self.banner['initrwgt'] += '\n<weightgroup type=\'mg_reweighting\'>\n'
320 for tag, diff in mg_rwgt_info:
321 self.banner['initrwgt'] += '<weight id=\'mg_reweight_%s\'>%s</weight>\n' % \
322 (tag, diff)
323 self.banner['initrwgt'] += '\n</weightgroup>\n'
324 self.banner['initrwgt'] = self.banner['initrwgt'].replace('\n\n', '\n')
325
326 output = open( self.lhe_input.name +'rw', 'w')
327
328
329 logger.info('starts to compute weight for events with the following modification to the param_card:')
330 logger.info(card_diff)
331
332
333 self.banner.write(output, close_tag=False)
334
335 if self.mother:
336 out_path = pjoin(self.mother.me_dir, 'Events', 'reweight.lhe')
337 output2 = open(out_path, 'w')
338 self.banner.write(output2, close_tag=False)
339 new_banner = banner.Banner(self.banner)
340 if not hasattr(self, 'run_card'):
341 self.run_card = new_banner.charge_card('run_card')
342 self.run_card['run_tag'] = 'reweight_%s' % rewgtid
343 new_banner['slha'] = s_new
344 del new_banner['initrwgt']
345
346 assert new_banner['slha'] != self.banner['slha']
347 assert 'initrwgt' in self.banner
348 ff = open(pjoin(self.mother.me_dir,'Events',self.mother.run_name, '%s_%s_banner.txt' % \
349 (self.mother.run_name, self.run_card['run_tag'])),'w')
350 new_banner.write(ff)
351 ff.close()
352
353
354 tag_name = 'mg_reweight_%s' % rewgtid
355 start = time.time()
356 cross = 0
357
358 os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'y'
359 if self.lhe_input.closed:
360 self.lhe_input = lhe_parser.EventFile(self.lhe_input.name)
361 self.lhe_input.seek(0)
362 for event_nb,event in enumerate(self.lhe_input):
363
364 if (event_nb % max(int(10**int(math.log10(float(event_nb)+1))),1000)==0):
365 running_time = misc.format_timer(time.time()-start)
366 logger.info('Event nb %s %s' % (event_nb, running_time))
367 if (event_nb==10001): logger.info('reducing number of print status. Next status update in 10000 events')
368
369
370 weight = self.calculate_weight(event)
371 cross += weight
372 event.reweight_data[tag_name] = weight
373
374 output.write(str(event))
375 if self.mother:
376 event.wgt = weight
377 event.reweight_data = {}
378 output2.write(str(event))
379
380 running_time = misc.format_timer(time.time()-start)
381 logger.info('All event done (nb_event: %s) %s' % (event_nb+1, running_time))
382
383 output.write('</LesHouchesEvents>\n')
384 output.close()
385 os.environ['GFORTRAN_UNBUFFERED_ALL'] = 'n'
386 if self.mother:
387 output2.write('</LesHouchesEvents>\n')
388 output2.close()
389
390 if hasattr(self.mother, 'results'):
391 run_name = self.mother.run_name
392 results = self.mother.results
393 results.add_run(run_name, self.run_card, current=True)
394 results.add_detail('nb_event', event_nb+1)
395 results.add_detail('cross', cross)
396 results.add_detail('error', 'nan')
397 self.mother.create_plot(mode='reweight', event_path=output2.name,
398 tag=self.run_card['run_tag'])
399
400 if 'plot' in results.current.reweight:
401 html_dir = pjoin(self.mother.me_dir, 'HTML', run_name)
402 td = pjoin(self.mother.options['td_path'], 'td')
403 MA = pjoin(self.mother.options['madanalysis_path'])
404 path1 = pjoin(html_dir, 'plots_parton')
405 path2 = pjoin(html_dir, 'plots_%s' % self.run_card['run_tag'])
406 outputplot = path2
407 combine_plots.merge_all_plots(path2, path1, outputplot, td, MA)
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422 self.lhe_input.close()
423 files.mv(output.name, self.lhe_input.name)
424 logger.info('Event %s have now the additional weight' % self.lhe_input.name)
425 logger.info('new cross-section is : %g pb' % cross)
426 self.terminate_fortran_executables(new_card_only=True)
427
428 self.all_cross_section[rewgtid] = cross
429
430
431
432
440
441
443 """routine to return the matrix element"""
444
445 tag, order = event.get_tag_and_order()
446 orig_order, Pdir = self.id_to_path[tag]
447
448 run_id = (tag, hypp_id)
449
450 if run_id in self.calculator:
451 external = self.calculator[run_id]
452 self.calculator_nbcall[run_id] += 1
453 else:
454
455
456 tmpdir = pjoin(self.me_dir,'rw_me', 'SubProcesses', Pdir)
457 executable_prod="./check"
458 if not os.path.exists(pjoin(tmpdir, 'check')):
459 misc.compile( cwd=tmpdir)
460 external = Popen(executable_prod, stdout=PIPE, stdin=PIPE,
461 stderr=STDOUT, cwd=tmpdir)
462 self.calculator[run_id] = external
463 self.calculator_nbcall[run_id] = 1
464
465 if hypp_id == 1:
466 external.stdin.write('param_card.dat\n')
467 elif hypp_id == 0:
468 external.stdin.write('param_card_orig.dat\n')
469
470 external.stdin.write('%g\n' % event.aqcd)
471 stdin_text = event.get_momenta_str(orig_order)
472 external.stdin.write(stdin_text)
473 me_value = external.stdout.readline()
474 try:
475 me_value = float(me_value)
476 except Exception:
477 print 'ZERO DETECTED'
478 print stdin_text
479 print me_value
480 os.system('lsof -p %s' % external.pid)
481 me_value = 0
482
483 if len(self.calculator) > 100:
484 logger.debug('more than 100 calculator. Perform cleaning')
485 nb_calls = self.calculator_nbcall.values()
486 nb_calls.sort()
487 cut = max([nb_calls[len(nb_calls)//2], 0.001 * nb_calls[-1]])
488 for key, external in list(self.calculator.items()):
489 nb = self.calculator_nbcall[key]
490 if nb < cut:
491 external.stdin.close()
492 external.stdout.close()
493 external.terminate()
494 del self.calculator[key]
495 del self.calculator_nbcall[key]
496 else:
497 self.calculator_nbcall[key] = self.calculator_nbcall[key] //10
498
499 return me_value
500
502 """routine to terminate all fortran executables"""
503
504 for (mode, production) in dict(self.calculator):
505
506 if new_card_only and production == 0:
507 continue
508 external = self.calculator[(mode, production)]
509 external.stdin.close()
510 external.stdout.close()
511 external.terminate()
512 del self.calculator[(mode, production)]
513
531
534
535
537 """Adding one element to the list based on the matrix element"""
538
539
540
541
542
543
544 @misc.mute_logger()
546 """generate the various directory for the weight evaluation"""
547
548
549 path_me = self.me_dir
550 try:
551 shutil.rmtree(pjoin(path_me,'rw_me'))
552 except Exception:
553 pass
554
555
556 complex_mass = False
557 has_cms = re.compile(r'''set\s+complex_mass_scheme\s*(True|T|1|true|$|;)''')
558 for line in self.banner.proc_card:
559 if line.startswith('set'):
560 self.mg5cmd.exec_cmd(line, printcmd=False, precmd=False, postcmd=False)
561 if has_cms.search(line):
562 complex_mass = True
563 elif line.startswith('define'):
564 try:
565 self.mg5cmd.exec_cmd(line, printcmd=False, precmd=False, postcmd=False)
566 except Exception:
567 pass
568
569 info = self.banner.get('proc_card', 'full_model_line')
570 if '-modelname' in info:
571 mg_names = False
572 else:
573 mg_names = True
574 model_name = self.banner.get('proc_card', 'model')
575 if model_name:
576 self.load_model(model_name, mg_names, complex_mass)
577 else:
578 raise self.InvalidCmd('Only UFO model can be loaded in this module.')
579
580 mgcmd = self.mg5cmd
581 modelpath = self.model.get('modelpath')
582 if os.path.basename(modelpath) != mgcmd._curr_model['name']:
583 name, restrict = mgcmd._curr_model['name'].rsplit('-',1)
584 if os.path.exists(pjoin(os.path.dirname(modelpath),name, 'restrict_%s.dat' % restrict)):
585 modelpath = pjoin(os.path.dirname(modelpath), mgcmd._curr_model['name'])
586
587 commandline="import model %s " % modelpath
588 mgcmd.exec_cmd(commandline)
589
590
591 processes = [line[9:].strip() for line in self.banner.proc_card
592 if line.startswith('generate')]
593 processes += [' '.join(line.split()[2:]) for line in self.banner.proc_card
594 if re.search('^\s*add\s+process', line)]
595 mgcmd.exec_cmd("set group_subprocesses False")
596
597 logger.info('generating the square matrix element for reweighting')
598 start = time.time()
599 commandline=''
600 for proc in processes:
601 if '[' not in proc:
602 commandline+="add process %s ;" % proc
603 else:
604 raise self.InvalidCmd('NLO processes can\'t be reweight')
605
606 commandline = commandline.replace('add process', 'generate',1)
607 logger.info(commandline)
608 mgcmd.exec_cmd(commandline, precmd=True)
609 commandline = 'output standalone_rw %s' % pjoin(path_me,'rw_me')
610 mgcmd.exec_cmd(commandline, precmd=True)
611 logger.info('Done %.4g' % (time.time()-start))
612 self.has_standalone_dir = True
613
614
615
616 matrix_elements = mgcmd._curr_matrix_elements.get_matrix_elements()
617
618 self.id_to_path = {}
619 for me in matrix_elements:
620 for proc in me.get('processes'):
621 initial = []
622 final = [l.get('id') for l in proc.get('legs')\
623 if l.get('state') or initial.append(l.get('id'))]
624 order = (initial, final)
625 tag = proc.get_initial_final_ids()
626 decay_finals = proc.get_final_ids_after_decay()
627
628 if tag[1] != decay_finals:
629 order = (initial, list(decay_finals))
630 decay_finals.sort()
631 tag = (tag[0], tuple(decay_finals))
632 Pdir = pjoin(path_me, 'rw_me', 'SubProcesses',
633 'P%s' % me.get('processes')[0].shell_string())
634 assert os.path.exists(Pdir), "Pdir %s do not exists" % Pdir
635 if tag in self.id_to_path:
636 if not Pdir == self.id_to_path[tag][1]:
637 misc.sprint(tag, Pdir, self.id_to_path[tag][1])
638 raise self.InvalidCmd, '2 different process have the same final states. This module can not handle such situation'
639 else:
640 continue
641 self.id_to_path[tag] = [order, Pdir]
642
643
644 - def load_model(self, name, use_mg_default, complex_mass=False):
664