From ea332f075f47bc028eb65f9b0b05fa19d40e672b Mon Sep 17 00:00:00 2001 From: Patrick de Perio Date: Fri, 17 Feb 2017 18:24:34 -0600 Subject: [PATCH 01/21] First attempt at synchronizing with pax_v6.4.0 https://github.com/XENON1T/pax/pull/516 --- .../fax_waveform/BatchMergeTruthAndProcessed.py | 7 ++----- montecarlo/fax_waveform/TruthSorting.py | 17 +++++++---------- montecarlo/fax_waveform/TruthSorting_arrays.py | 13 +++++-------- montecarlo/run_sim.sh | 3 +-- 4 files changed, 15 insertions(+), 25 deletions(-) diff --git a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py index 66bea60..f2f2ce8 100644 --- a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py +++ b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py @@ -14,7 +14,7 @@ if len(sys.argv)<=1: print("======== Usage =========") - print("python BatchMergeTruthAndProcessed.py <(opt)top-to-total fraction in truth(default 0.68)> <(opt)relative path for submission> <(opt) if use public node (1) optional (2 for use kicp nodes)> <(opt)Submit ID> <(opt) if use arrays in output (1) (default 0)>") + print("python BatchMergeTruthAndProcessed.py <(opt)relative path for submission> <(opt) if use public node (1) optional (2 for use kicp nodes)> <(opt)Submit ID> <(opt) if use arrays in output (1) (default 0)>") exit() CurrentEXE = sys.argv[0] @@ -23,9 +23,6 @@ ProcessedRootPath = sys.argv[3] OutputPath = sys.argv[4] IfPublicNode = 1 -TopFraction = 0.68 -if len(sys.argv)>5: - TopFraction = float(sys.argv[5]) RelativeSubmitPath = "Submit" if len(sys.argv)>6: RelativeSubmitPath = sys.argv[6] @@ -112,7 +109,7 @@ subp.call("echo '#SBATCH --qos=xenon1t-kicp\n' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True) subp.call("echo '. /home/mcfate/Env/GlobalPAXEnv.sh\n\n' >> "+SubmitFile, shell=True) - subp.call("echo 'python "+EXE1+" "+TruthRootFilename+" "+TmpOutputFilename+" "+str(TopFraction)+"' >> "+SubmitFile, shell=True) + subp.call("echo 'python "+EXE1+" "+TruthRootFilename+" "+TmpOutputFilename+"' >> "+SubmitFile, shell=True) subp.call("echo 'python "+EXE2+" "+AbsoluteConfigFile+" "+TmpOutputFilename+" "+ProcessedRootFilename+" "+OutputFilename+"' >> "+SubmitFile, shell=True) #submit diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index 64106d5..5a1f086 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -23,23 +23,19 @@ if len(sys.argv)<2: print("============= Syntax =============") - print("python TruthSorting.py <(opt) top-to-total fraction; default = 0.68> ") + print("python TruthSorting.py ") exit() TruthFile = sys.argv[1] OutputFile = sys.argv[2] OutputFile = OutputFile.split('.')[0] -mean_top_fraction = 0.68 -if len(sys.argv)>3: - mean_top_fraction = float(sys.argv[3]) OutputFormat=0 if len(sys.argv)>4: OutputFormat = float(sys.argv[4]) print ("Input file: ", TruthFile) -print ("Mean top fraction: ", mean_top_fraction) ################# ## load the root files @@ -71,7 +67,7 @@ Data['s2_first_electron_time_truth'] = [] Data['s2_time_std_truth'] = [] Data['s2_area_truth'] = [] -Data['s2_bottom_area_truth'] = [] +Data['peak_top_fraction'] = [] Data['x_truth'] = [] Data['y_truth'] = [] @@ -90,7 +86,7 @@ s2_time_truth = -1 s2_time_std_truth = -1 s2_area_truth = -1 - s2_bottom_area_truth = -1 + peak_top_fraction = -1 x_truth = -1e10 y_truth = -1e10 while truth_tree.event==event_id: @@ -102,14 +98,15 @@ s1_time_truth = truth_tree.t_mean_photons s1_time_std_truth = truth_tree.t_sigma_photons s1_area_truth = truth_tree.n_photons - else: + peak_top_fraction = truth_tree.peak_top_fraction + else: #print("Iterator: "+str(iteration_id)+" -> S2") s2_electron_time_truth = truth_tree.t_mean_electrons s2_first_electron_time_truth = truth_tree.t_first_electron s2_time_truth = truth_tree.t_mean_photons s2_time_std_truth = truth_tree.t_sigma_photons s2_area_truth = truth_tree.n_photons - s2_bottom_area_truth = truth_tree.n_photons * (1.-mean_top_fraction) + peak_top_fraction = truth_tree.peak_top_fraction x_truth = truth_tree.x y_truth = truth_tree.y iteration_id += 1 @@ -125,7 +122,7 @@ Data['s2_time_truth'].append(s2_time_truth) Data['s2_time_std_truth'].append(s2_time_std_truth) Data['s2_area_truth'].append(s2_area_truth) - Data['s2_bottom_area_truth'].append(s2_bottom_area_truth) + Data['peak_top_fraction'].append(peak_top_fraction) Data['x_truth'].append(x_truth) Data['y_truth'].append(y_truth) diff --git a/montecarlo/fax_waveform/TruthSorting_arrays.py b/montecarlo/fax_waveform/TruthSorting_arrays.py index e5b1935..98115ca 100644 --- a/montecarlo/fax_waveform/TruthSorting_arrays.py +++ b/montecarlo/fax_waveform/TruthSorting_arrays.py @@ -22,23 +22,19 @@ if len(sys.argv)<2: print("============= Syntax =============") - print("python TruthSorting_arrays.py <(opt) top-to-total fraction; default = 0.68> ") + print("python TruthSorting_arrays.py ") exit() TruthFile = sys.argv[1] OutputFile = sys.argv[2] OutputFile = OutputFile.split('.')[0] -mean_top_fraction = 0.68 -if len(sys.argv)>3: - mean_top_fraction = float(sys.argv[3]) OutputFormat=0 if len(sys.argv)>4: OutputFormat = float(sys.argv[4]) print ("Input file: ", TruthFile) -print ("Mean top fraction: ", mean_top_fraction) ################# ## load the root files @@ -62,8 +58,8 @@ # initialize Data for truth event_keys = ['index_truth', 'peaks_length'] -s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth'] -s2_only_keys = ['electron_time_truth', 'first_electron_time_truth', 'last_electron_time_truth', 'bottom_area_truth'] +s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth', 'peak_top_fraction'] +s2_only_keys = ['electron_time_truth', 'first_electron_time_truth', 'last_electron_time_truth'] for field in (event_keys + s1s2_keys + s2_only_keys): Data[field] = [] @@ -96,6 +92,8 @@ result['x_truth'].append(truth_tree.x) result['y_truth'].append(truth_tree.y) result['z_truth'].append(truth_tree.z) + result['peak_top_fraction'].append(truth_tree.peak_top_fraction) + if tag==0: # peak is an S1 for s2_field in s2_only_keys: @@ -105,7 +103,6 @@ result['electron_time_truth'].append(truth_tree.t_mean_electrons) result['first_electron_time_truth'].append(truth_tree.t_first_electron) result['last_electron_time_truth'].append(truth_tree.t_last_electron) - result['bottom_area_truth'].append(truth_tree.n_photons * (1.-mean_top_fraction)) iteration_id += 1 if iteration_id>=NumStepsInTruth: break diff --git a/montecarlo/run_sim.sh b/montecarlo/run_sim.sh index edbf9d9..a982443 100755 --- a/montecarlo/run_sim.sh +++ b/montecarlo/run_sim.sh @@ -267,9 +267,8 @@ fi # Flatten fax truth info FAXSORT_FILENAME=${FAX_FILENAME}_sort -MEAN_TOP_FRACTION=0.68 # To be improved: https://github.com/XENON1T/mc/issues/20 FAXSORT_OUTPUT_FORMAT=2 # Pickle + ROOT -(time python ${CVMFSDIR}/releases/processing/montecarlo/fax_waveform/TruthSorting_arrays.py ${FAX_FILENAME}.root ${FAXSORT_FILENAME} ${MEAN_TOP_FRACTION} ${FAXSORT_OUTPUT_FORMAT};) 2>&1 | tee ${FAXSORT_FILENAME}.log +(time python ${CVMFSDIR}/releases/processing/montecarlo/fax_waveform/TruthSorting_arrays.py ${FAX_FILENAME}.root ${FAXSORT_FILENAME} ${FAXSORT_OUTPUT_FORMAT};) 2>&1 | tee ${FAXSORT_FILENAME}.log if [ $? -ne 0 ]; then terminate 16 From f9ac0466ed1e5c0230dc5e887aff2b0d8b74c3d4 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Fri, 17 Feb 2017 18:50:24 -0600 Subject: [PATCH 02/21] Temporary Truth sorting --- montecarlo/fax_waveform/TruthSorting.py | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index 5a1f086..cd9d37b 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -62,12 +62,13 @@ Data['s1_time_truth'] = [] Data['s1_time_std_truth'] = [] Data['s1_area_truth'] = [] +Data['s1_area_top_fraction_truth'] = [] Data['s2_time_truth'] = [] Data['s2_electron_time_truth'] = [] Data['s2_first_electron_time_truth'] = [] Data['s2_time_std_truth'] = [] Data['s2_area_truth'] = [] -Data['peak_top_fraction'] = [] +Data['s2_area_top_fraction_truth'] = [] Data['x_truth'] = [] Data['y_truth'] = [] @@ -81,32 +82,39 @@ s1_time_truth = -1 s1_time_std_truth = -1 s1_area_truth = -1 + s1_area_top_fraction_truth = -1 s2_electron_time_truth = -1 s2_first_electron_time_truth = -1 s2_time_truth = -1 s2_time_std_truth = -1 s2_area_truth = -1 - peak_top_fraction = -1 + s2_area_top_fraction_truth = -1 x_truth = -1e10 y_truth = -1e10 + ifcounteds2 = 0 while truth_tree.event==event_id: - tag = 0 # 0 for s1, 1 for s2 - if not str(truth_tree.n_electrons)=='nan': + tag = 0 # 0 for s1, 1 for s2, 2 for photoionization + if not str(truth_tree.n_electrons)=='nan' and ifcounteds2==0: tag = 1 + ifcounteds2==1 + elif ifcounteds2!=0: + tag=0 + else: + tag=2 if tag==0: #print("Iterator: "+str(iteration_id)+" -> S1") s1_time_truth = truth_tree.t_mean_photons s1_time_std_truth = truth_tree.t_sigma_photons s1_area_truth = truth_tree.n_photons - peak_top_fraction = truth_tree.peak_top_fraction - else: + s1_area_top_fraction_truth = truth_tree.peak_top_fraction + elif tag==1: #print("Iterator: "+str(iteration_id)+" -> S2") s2_electron_time_truth = truth_tree.t_mean_electrons s2_first_electron_time_truth = truth_tree.t_first_electron s2_time_truth = truth_tree.t_mean_photons s2_time_std_truth = truth_tree.t_sigma_photons s2_area_truth = truth_tree.n_photons - peak_top_fraction = truth_tree.peak_top_fraction + s2_area_top_fraction_truth = truth_tree.peak_top_fraction x_truth = truth_tree.x y_truth = truth_tree.y iteration_id += 1 From 5573dfda2c8ff2c58b66d2df95b29b8fe3e9edc6 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Sat, 18 Feb 2017 17:11:51 -0600 Subject: [PATCH 03/21] Updating TruthSorting.py according to new fax update --- montecarlo/fax_waveform/TruthSorting.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index cd9d37b..daadd12 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -91,14 +91,14 @@ s2_area_top_fraction_truth = -1 x_truth = -1e10 y_truth = -1e10 - ifcounteds2 = 0 + ifcounteds1 = 0 while truth_tree.event==event_id: tag = 0 # 0 for s1, 1 for s2, 2 for photoionization - if not str(truth_tree.n_electrons)=='nan' and ifcounteds2==0: + if not str(truth_tree.n_electrons)=='nan': tag = 1 - ifcounteds2==1 - elif ifcounteds2!=0: + elif ifcounteds1==0: tag=0 + ifcounteds1=1 else: tag=2 if tag==0: @@ -130,7 +130,8 @@ Data['s2_time_truth'].append(s2_time_truth) Data['s2_time_std_truth'].append(s2_time_std_truth) Data['s2_area_truth'].append(s2_area_truth) - Data['peak_top_fraction'].append(peak_top_fraction) + Data['s1_area_top_fraction_truth'].append(s1_area_top_fraction_truth) + Data['s2_area_top_fraction_truth'].append(s2_area_top_fraction_truth) Data['x_truth'].append(x_truth) Data['y_truth'].append(y_truth) From b3b27ae0c24124a6e525aceae6ff50321e088150 Mon Sep 17 00:00:00 2001 From: Joey Howlett Date: Sat, 18 Feb 2017 17:26:35 -0600 Subject: [PATCH 04/21] fixed array capabilities for new truth info --- .../fax_waveform/BatchMergeTruthAndProcessed.py | 14 +++++++------- montecarlo/fax_waveform/TruthSorting_arrays.py | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py index f2f2ce8..5949f24 100644 --- a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py +++ b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py @@ -24,16 +24,16 @@ OutputPath = sys.argv[4] IfPublicNode = 1 RelativeSubmitPath = "Submit" +if len(sys.argv)>5: + RelativeSubmitPath = sys.argv[5] if len(sys.argv)>6: - RelativeSubmitPath = sys.argv[6] -if len(sys.argv)>7: - IfPublicNode = int(sys.argv[7]) + IfPublicNode = int(sys.argv[6]) SubmitID = 0 -if len(sys.argv)>8: - SubmitID = int(sys.argv[8]) +if len(sys.argv)>7: + SubmitID = int(sys.argv[7]) ArrayOutput = 0 -if len(sys.argv)>9: - ArrayOutput = int(sys.argv[9]) +if len(sys.argv)>8: + ArrayOutput = int(sys.argv[8]) ####################### diff --git a/montecarlo/fax_waveform/TruthSorting_arrays.py b/montecarlo/fax_waveform/TruthSorting_arrays.py index 98115ca..8df7e1d 100644 --- a/montecarlo/fax_waveform/TruthSorting_arrays.py +++ b/montecarlo/fax_waveform/TruthSorting_arrays.py @@ -58,7 +58,7 @@ # initialize Data for truth event_keys = ['index_truth', 'peaks_length'] -s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth', 'peak_top_fraction'] +s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth', 'top_fraction'] s2_only_keys = ['electron_time_truth', 'first_electron_time_truth', 'last_electron_time_truth'] for field in (event_keys + s1s2_keys + s2_only_keys): @@ -92,7 +92,7 @@ result['x_truth'].append(truth_tree.x) result['y_truth'].append(truth_tree.y) result['z_truth'].append(truth_tree.z) - result['peak_top_fraction'].append(truth_tree.peak_top_fraction) + result['top_fraction'].append(truth_tree.top_fraction) if tag==0: # peak is an S1 From 63c664f8f1437fc9a4dacb8a8446ebb5fdfe0d2f Mon Sep 17 00:00:00 2001 From: Joey Howlett Date: Sat, 18 Feb 2017 17:29:51 -0600 Subject: [PATCH 05/21] fixed peak_top_fraction conflict --- montecarlo/fax_waveform/TruthSorting.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index daadd12..41efe77 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -106,7 +106,7 @@ s1_time_truth = truth_tree.t_mean_photons s1_time_std_truth = truth_tree.t_sigma_photons s1_area_truth = truth_tree.n_photons - s1_area_top_fraction_truth = truth_tree.peak_top_fraction + s1_area_top_fraction_truth = truth_tree.top_fraction elif tag==1: #print("Iterator: "+str(iteration_id)+" -> S2") s2_electron_time_truth = truth_tree.t_mean_electrons @@ -114,7 +114,7 @@ s2_time_truth = truth_tree.t_mean_photons s2_time_std_truth = truth_tree.t_sigma_photons s2_area_truth = truth_tree.n_photons - s2_area_top_fraction_truth = truth_tree.peak_top_fraction + s2_area_top_fraction_truth = truth_tree.top_fraction x_truth = truth_tree.x y_truth = truth_tree.y iteration_id += 1 From efbc109f28f348d37eacfc8e73f5fa668e395d17 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Mon, 20 Feb 2017 13:52:39 -0600 Subject: [PATCH 06/21] Fixed some bugs of TruthSorting* --- montecarlo/fax_waveform/TruthSorting.py | 6 +++--- montecarlo/fax_waveform/TruthSorting_arrays.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index 41efe77..59dcba8 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -32,8 +32,8 @@ OutputFile = OutputFile.split('.')[0] OutputFormat=0 -if len(sys.argv)>4: - OutputFormat = float(sys.argv[4]) +if len(sys.argv)>3: + OutputFormat = float(sys.argv[3]) print ("Input file: ", TruthFile) @@ -114,7 +114,7 @@ s2_time_truth = truth_tree.t_mean_photons s2_time_std_truth = truth_tree.t_sigma_photons s2_area_truth = truth_tree.n_photons - s2_area_top_fraction_truth = truth_tree.top_fraction + s2_area_top_fraction_truth = truth_tree.top_fraction x_truth = truth_tree.x y_truth = truth_tree.y iteration_id += 1 diff --git a/montecarlo/fax_waveform/TruthSorting_arrays.py b/montecarlo/fax_waveform/TruthSorting_arrays.py index 8df7e1d..863246f 100644 --- a/montecarlo/fax_waveform/TruthSorting_arrays.py +++ b/montecarlo/fax_waveform/TruthSorting_arrays.py @@ -31,8 +31,8 @@ OutputFile = OutputFile.split('.')[0] OutputFormat=0 -if len(sys.argv)>4: - OutputFormat = float(sys.argv[4]) +if len(sys.argv)>3: + OutputFormat = float(sys.argv[3]) print ("Input file: ", TruthFile) From 57285f4975571269bc85d549bf3bd06a4e4c09a6 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Mon, 20 Feb 2017 15:36:21 -0600 Subject: [PATCH 07/21] No midpoint anymore in pax processed. Switch to using center_time for S1&s2 peak time --- montecarlo/fax_waveform/ReduceDataNormal.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/montecarlo/fax_waveform/ReduceDataNormal.py b/montecarlo/fax_waveform/ReduceDataNormal.py index 110bfe4..47f87a9 100644 --- a/montecarlo/fax_waveform/ReduceDataNormal.py +++ b/montecarlo/fax_waveform/ReduceDataNormal.py @@ -150,7 +150,7 @@ def extract_data(self, event): # This runs on each event # Look here for more info: http://xenon1t.github.io/pax/format.html#peak s1_fields = {'S1sTot': 'area', 'S1TopFraction': 'area_fraction_top', - 'S1sPeakTime': 'area_midpoint', + 'S1sPeakTime': 'center_time', 'S1sPeakTimeStd': 'hit_time_std', 'S1sCoin': 'n_contributing_channels', 'S1sHeight': 'height', @@ -158,7 +158,7 @@ def extract_data(self, event): # This runs on each event } s2_fields = {'S2sTot': 'area', 'S2TopFraction': 'area_fraction_top', - 'S2sPeakTime': 'area_midpoint', + 'S2sPeakTime': 'center_time', 'S2sPeakTimeStd': 'hit_time_std', 'S2sCoin': 'n_contributing_channels', 'S2sHeight': 'height', From dee8ae20d84c20409b70fd1c24c73ba1e6c87cd1 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Tue, 21 Feb 2017 13:47:29 -0600 Subject: [PATCH 08/21] Change BatchMergeTruthAndProcessed.py duration to 5min to increase the job priority. The merging of 1000 events usually only takes less than 1min --- montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py index 5949f24..24af151 100644 --- a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py +++ b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py @@ -99,7 +99,7 @@ subp.call("echo '#!/bin/bash\n' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --output="+SubmitPath+"/myout_"+str(SubmitID)+"_"+str(j)+".txt \n' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --error="+SubmitPath+"/myerr_"+str(SubmitID)+"_"+str(j)+".txt\n' >> "+SubmitFile, shell=True) - subp.call("echo '#SBATCH --time=01:19:00\n' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --time=00:05:00\n' >> "+SubmitFile, shell=True) if not IfPublicNode: subp.call("echo '#SBATCH --account=pi-lgrandi\n' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --qos=xenon1t\n' >> "+SubmitFile, shell=True) From b32fc7863e2dd66fb8b6dbbbf68dc04e745a6c00 Mon Sep 17 00:00:00 2001 From: Fei Gao Date: Tue, 21 Feb 2017 21:03:55 -0600 Subject: [PATCH 09/21] new createfakecsv.py --- montecarlo/fax_waveform/CreateFakeCSV.py | 44 ++++++++++++++++++++---- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/montecarlo/fax_waveform/CreateFakeCSV.py b/montecarlo/fax_waveform/CreateFakeCSV.py index a7f2019..ca05f5a 100644 --- a/montecarlo/fax_waveform/CreateFakeCSV.py +++ b/montecarlo/fax_waveform/CreateFakeCSV.py @@ -44,7 +44,7 @@ #################################### ## Some nuisance parameters (HARDCODE WARNING): #################################### -MaxDriftTime = 650. # us +MaxDriftTime = 675. # us #################################### @@ -65,7 +65,8 @@ def IfPassFV(x,y,z): if I<1: return True elif Detector == "XENON1T": # NEED TO UPDATE THIS - Zlower, Zupper = -90*scalecmtomm, -15*scalecmtomm + #Zlower, Zupper = -90*scalecmtomm, -15*scalecmtomm + Zlower, Zupper = -120*scalecmtomm, -1*scalecmtomm Zcut = ((z>=Zlower) & (z<=Zupper)) R2upper=radius2_cut(z) Rcut = (x**2+y**20 and DtTest Date: Tue, 21 Feb 2017 22:04:48 -0600 Subject: [PATCH 10/21] Remove previous commitment --- montecarlo/fax_waveform/CreateFakeCSV.py | 44 ++++-------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/montecarlo/fax_waveform/CreateFakeCSV.py b/montecarlo/fax_waveform/CreateFakeCSV.py index ca05f5a..a7f2019 100644 --- a/montecarlo/fax_waveform/CreateFakeCSV.py +++ b/montecarlo/fax_waveform/CreateFakeCSV.py @@ -44,7 +44,7 @@ #################################### ## Some nuisance parameters (HARDCODE WARNING): #################################### -MaxDriftTime = 675. # us +MaxDriftTime = 650. # us #################################### @@ -65,8 +65,7 @@ def IfPassFV(x,y,z): if I<1: return True elif Detector == "XENON1T": # NEED TO UPDATE THIS - #Zlower, Zupper = -90*scalecmtomm, -15*scalecmtomm - Zlower, Zupper = -120*scalecmtomm, -1*scalecmtomm + Zlower, Zupper = -90*scalecmtomm, -15*scalecmtomm Zcut = ((z>=Zlower) & (z<=Zupper)) R2upper=radius2_cut(z) Rcut = (x**2+y**20 and DtTest Date: Wed, 22 Feb 2017 10:57:20 -0600 Subject: [PATCH 11/21] fixed some bugs, added begin_production.py capability, added peak/s1s2 minitree capability --- .../BatchMergeTruthAndProcessed.py | 10 +- .../fax_waveform/BatchReduceDataSubmission.py | 19 ++- .../fax_waveform/Configs/PeakEfficiency | 13 ++ montecarlo/fax_waveform/Configs/basics_config | 18 +++ .../fax_waveform/Configs/s1s2_preserve_all | 57 +++++++ .../MergeTruthAndProcessed_peaks.py | 107 ++++++++++++ montecarlo/fax_waveform/README.md | 22 +++ montecarlo/fax_waveform/TruthSorting.py | 6 +- .../fax_waveform/TruthSorting_arrays.py | 5 +- montecarlo/fax_waveform/begin_production.py | 68 ++++++++ montecarlo/fax_waveform/logs/170221_test.log | 153 ++++++++++++++++++ .../fax_waveform/processed_dataset_list.dat | 5 + montecarlo/fax_waveform/reduce_peak_level.py | 53 ++++++ montecarlo/fax_waveform/run_fax.sh | 1 + montecarlo/fax_waveform/setup_production.py | 60 +++++++ .../fax_waveform/sort_processed_files.sh | 43 +++++ .../fax_waveform/submission_merge/desc.txt | 1 + .../fax_waveform/submission_reduce/desc.txt | 2 + 18 files changed, 632 insertions(+), 11 deletions(-) create mode 100644 montecarlo/fax_waveform/Configs/PeakEfficiency create mode 100644 montecarlo/fax_waveform/Configs/basics_config create mode 100644 montecarlo/fax_waveform/Configs/s1s2_preserve_all create mode 100644 montecarlo/fax_waveform/MergeTruthAndProcessed_peaks.py create mode 100644 montecarlo/fax_waveform/begin_production.py create mode 100644 montecarlo/fax_waveform/logs/170221_test.log create mode 100644 montecarlo/fax_waveform/processed_dataset_list.dat create mode 100644 montecarlo/fax_waveform/reduce_peak_level.py create mode 100644 montecarlo/fax_waveform/setup_production.py create mode 100755 montecarlo/fax_waveform/sort_processed_files.sh create mode 100644 montecarlo/fax_waveform/submission_merge/desc.txt create mode 100644 montecarlo/fax_waveform/submission_reduce/desc.txt diff --git a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py index 24af151..247e545 100644 --- a/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py +++ b/montecarlo/fax_waveform/BatchMergeTruthAndProcessed.py @@ -14,7 +14,7 @@ if len(sys.argv)<=1: print("======== Usage =========") - print("python BatchMergeTruthAndProcessed.py <(opt)relative path for submission> <(opt) if use public node (1) optional (2 for use kicp nodes)> <(opt)Submit ID> <(opt) if use arrays in output (1) (default 0)>") + print("python BatchMergeTruthAndProcessed.py <(opt)relative path for submission> <(opt) if use public node (1) optional (2 for use kicp nodes)> <(opt)Submit ID> <(opt) if use arrays in output (1) (default 0)> <(opt) minitree type; 0(default): Basics, 1: S1S2Properties, 2: PeakEfficiency") exit() CurrentEXE = sys.argv[0] @@ -34,6 +34,9 @@ ArrayOutput = 0 if len(sys.argv)>8: ArrayOutput = int(sys.argv[8]) +MinitreeType=0 +if len(sys.argv)>9: + MinitreeType = int(sys.argv[9]) ####################### @@ -49,7 +52,10 @@ EXE1 = CurrentPath+"/"+EXE_Path+"TruthSorting_arrays.py" else: EXE1 = CurrentPath+"/"+EXE_Path+"TruthSorting.py" -EXE2 = CurrentPath+"/"+EXE_Path+"MergeTruthAndProcessed.py" +if MinitreeType==2: + EXE2 = CurrentPath+"/"+EXE_Path+"MergeTruthAndProcessed_peaks.py" +else: + EXE2 = CurrentPath+"/"+EXE_Path+"MergeTruthAndProcessed.py" ####################### diff --git a/montecarlo/fax_waveform/BatchReduceDataSubmission.py b/montecarlo/fax_waveform/BatchReduceDataSubmission.py index e68242b..573fd0e 100644 --- a/montecarlo/fax_waveform/BatchReduceDataSubmission.py +++ b/montecarlo/fax_waveform/BatchReduceDataSubmission.py @@ -13,7 +13,7 @@ if len(sys.argv)<=1: print("======== Usage =========") - print("python ReduceDataNormal.py ") + print("python BatchReduceDataSubmission.py <(opt) minitree type; 1: S1S2Properties, 2: PeakEfficiency>") print("======== List file format: ==========") print("ex.:") print("FakeWaveform_XENON1T_000000_pax") @@ -35,13 +35,20 @@ SubmitID = 0 if len(sys.argv)>6: SubmitID = int(sys.argv[6]) +minitree_type = '1' +if len(sys.argv)>7: + minitree_type = sys.argv[7] + ########################## ## Some nuisance settings ########################## CurrentPath = os.getcwd() CurrentUser = getpass.getuser() -EXE = CurrentPath+"/"+EXE_Path+"/ReduceDataNormal.py" +if minitree_type == '1': + EXE = CurrentPath+"/"+EXE_Path+"/ReduceDataNormal.py" +elif minitree_type == '2': + EXE = CurrentPath+"/"+EXE_Path+"/reduce_peak_level.py" MaxNumJob = 64 if not IfPublicNode: MaxNumJob = 200 @@ -87,7 +94,10 @@ subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True) subp.call("echo '. /home/mcfate/Env/GlobalPAXEnv.sh\n\n' >> "+SubmitFile, shell=True) subp.call("echo 'python "+EXE+" "+filename+" "+DataPath+"' >> "+SubmitFile, shell=True) - subp.call("echo 'mv "+SubmitPath+"/"+filename+"_S1S2Properties.root "+OutputPath+"' >> "+SubmitFile, shell=True) + if minitree_type=='1': + subp.call("echo 'mv "+SubmitPath+"/"+filename+"_S1S2Properties.root "+OutputPath+"' >> "+SubmitFile, shell=True) + elif minitree_type=='2': + subp.call("echo 'mv "+SubmitPath+"/"+filename+"_PeakEfficiency.root "+OutputPath+"' >> "+SubmitFile, shell=True) #submit IfSubmitted=0 @@ -114,6 +124,3 @@ else: time.sleep(30) - - - diff --git a/montecarlo/fax_waveform/Configs/PeakEfficiency b/montecarlo/fax_waveform/Configs/PeakEfficiency new file mode 100644 index 0000000..b65dcb9 --- /dev/null +++ b/montecarlo/fax_waveform/Configs/PeakEfficiency @@ -0,0 +1,13 @@ +Name: PeakEfficiency +event_number event_number_processed +run_number run_number +area area +hit_time_mean hit_time_mean +hit_time_std hit_time_std +type type +area_fraction_top area_fraction_top +height height +n_contributing_channels n_contributing_channels +range_50p_area range_50p_area +range_70p_area range_70p_area +range_90p_area range_90p_area diff --git a/montecarlo/fax_waveform/Configs/basics_config b/montecarlo/fax_waveform/Configs/basics_config new file mode 100644 index 0000000..ce91d28 --- /dev/null +++ b/montecarlo/fax_waveform/Configs/basics_config @@ -0,0 +1,18 @@ +Name: Basics +cs1 cs1_processed +cs2 cs2_processed +drift_time drift_time_processed +event_number event_number_processed +largest_coincidence largest_coincidence_processed +largest_other_s1 largest_other_s1_processed +largest_other_s2 largest_other_s2_processed +run_number run_number_processed +s1 s1_processed +s1_area_fraction_top s1_area_fraction_top_processed +s1_range_50p_area s1_range_50p_area_processed +s2 s2_processed +s2_area_fraction_top s2_area_fraction_top_processed +s2_range_50p_area s2_range_50p_area_processed +x x_processed +y y_processed +z z_processed diff --git a/montecarlo/fax_waveform/Configs/s1s2_preserve_all b/montecarlo/fax_waveform/Configs/s1s2_preserve_all new file mode 100644 index 0000000..3688c02 --- /dev/null +++ b/montecarlo/fax_waveform/Configs/s1s2_preserve_all @@ -0,0 +1,57 @@ +Name: S1S2Properties +index index +NbPeaks NbPeaks +NbS1s NbS1s +NbS2s NbS2s +S1PatternLnL S1PatternLnL +S1S2InMainInteraction S1S2InMainInteraction +S1TopFraction S1TopFraction +S1_X_PosRecMaxPMT S1_X_PosRecMaxPMT +S1_X_PosRecNeuralNet S1_X_PosRecNeuralNet +S1_X_PosRecRobustWeightedMean S1_X_PosRecRobustWeightedMean +S1_X_PosRecTopPatternFit S1_X_PosRecTopPatternFit +S1_X_PosRecWeightedSum S1_X_PosRecWeightedSum +S1_Y_PosRecMaxPMT S1_Y_PosRecMaxPMT +S1_Y_PosRecNeuralNet S1_Y_PosRecNeuralNet +S1_Y_PosRecRobustWeightedMean S1_Y_PosRecRobustWeightedMean +S1_Y_PosRecTopPatternFit S1_Y_PosRecTopPatternFit +S1_Y_PosRecWeightedSum S1_Y_PosRecWeightedSum +S1sCoin S1sCoin +S1sCorrection S1sCorrection +S1sHeight S1sHeight +S1sLowWidth S1sLowWidth +S1sNbSaturationChannels S1sNbSaturationChannels +S1sPeakTime S1sPeakTime +S1sPeakTimeStd S1sPeakTimeStd +S1sTot S1sTot +S1sTotGained S1sTotGained +S1sTotSecond S1sTotSecond +S1sTotSecondGained S1sTotSecondGained +S1sWidth S1sWidth +S2PosGoodnessOfFit S2PosGoodnessOfFit +S2TopFraction S2TopFraction +S2_X_PosRecMaxPMT S2_X_PosRecMaxPMT +S2_X_PosRecNeuralNet S2_X_PosRecNeuralNet +S2_X_PosRecRobustWeightedMean S2_X_PosRecRobustWeightedMean +S2_X_PosRecTopPatternFit S2_X_PosRecTopPatternFit +S2_X_PosRecWeightedSum S2_X_PosRecWeightedSum +S2_Y_PosRecMaxPMT S2_Y_PosRecMaxPMT +S2_Y_PosRecNeuralNet S2_Y_PosRecNeuralNet +S2_Y_PosRecRobustWeightedMean S2_Y_PosRecRobustWeightedMean +S2_Y_PosRecTopPatternFit S2_Y_PosRecTopPatternFit +S2_Y_PosRecWeightedSum S2_Y_PosRecWeightedSum +S2sCoin S2sCoin +S2sCorrection S2sCorrection +S2sHeight S2sHeight +S2sLowWidth S2sLowWidth +S2sNbSaturationChannels S2sNbSaturationChannels +S2sPeakTime S2sPeakTime +S2sPeakTimeStd S2sPeakTimeStd +S2sTot S2sTot +S2sTotGained S2sTotGained +S2sTotSecond S2sTotSecond +S2sTotSecondGained S2sTotSecondGained +S2sWidth S2sWidth +event_number event_number +run_number run_number +time time diff --git a/montecarlo/fax_waveform/MergeTruthAndProcessed_peaks.py b/montecarlo/fax_waveform/MergeTruthAndProcessed_peaks.py new file mode 100644 index 0000000..ea60cfa --- /dev/null +++ b/montecarlo/fax_waveform/MergeTruthAndProcessed_peaks.py @@ -0,0 +1,107 @@ +########################### +## Code for merging the truth & processed info from simulated data +## by Qing Lin +## 1) Truth info from the output pickle by TruthSorting.py +## 2) Processed info from any minitree, but need to specify which branches to keep in the merged file +## Notice that the external file is needed for this specification +## And for the convenience of the user, it is better to name the branch in output the same way +######### +## @ 2017-01-09 +## Please NOTE the code can only be used with single S1&S2 simulation +## Double/Multiple peak simulation is not available in merging/minitree yet +########################### + +import pickle +import pandas as pd + +import ROOT +from ROOT import TFile +from ROOT import TTree + +import sys + + +if len(sys.argv)<2: + print("============= Syntax =============") + print("python MergeTruthAndProcessed.py ") + exit() + + +ConfigFile = sys.argv[1] +TruthFile = sys.argv[2] +ProcessedFile = sys.argv[3] +OutputFile = sys.argv[4] + +################# +## load the config file +################# +fin = open(ConfigFile) +lines = fin.readlines() +fin.close() + +processedTreeName = "" +BranchesToKeep = [['index', 'index_processed']] # default put index there for merging + +for i, line in enumerate(lines): + if i==0: + contents = line[:-1].split('Name: ') + if len(contents)<=1: + raise ValueError("Tree name not properly defined") + processedTreeName = contents[1] + else: + line = line.replace("\t", " ") + line = line.replace("\n", " ") + contents = line.split(" ") + for content in list(contents): + if content=='': + contents.remove(content) + BranchesToKeep.append( + [ + contents[0], + contents[1], + ] + ) + + + +################# +## load the input files +## and pandas and TTrees +################# +truthData = pickle.load( open(TruthFile, 'rb') ) + +pfile2 = TFile(ProcessedFile) +processed_tree = pfile2.Get(processedTreeName) + +#if (not processed_tree): +# raise ValueError("Input file not complete") + +#NumEventsInData = processed_tree.GetEntries() +NumStepsInTruth = 0 +for i, item in enumerate(truthData): + NumStepsInTruth = int(len(truthData[item])) + if i==0: + break + + +################### +## load the data dict from processed root file +################### +#df = pd.DataFrame(processedPandasData) +import root_numpy +df = pd.DataFrame.from_records(root_numpy.root2rec(ProcessedFile)) +for branch_name in list(df): + if ('length' in branch_name) and (branch_name != 'peaks_length'): + df.drop(branch_name, 1) + +##################### +## Merge the new dictionary to existing dataframe +##################### + +df = df.merge(truthData, left_on='event_number', right_on='index_truth', how='outer') + +####################### +## Save to pickle +####################### +#pickle.dump(df, open(OutputFile, 'wb')) +df.to_pickle(OutputFile) diff --git a/montecarlo/fax_waveform/README.md b/montecarlo/fax_waveform/README.md index f928faf..f0a3af2 100644 --- a/montecarlo/fax_waveform/README.md +++ b/montecarlo/fax_waveform/README.md @@ -6,3 +6,25 @@ In order to run the fax code: - checkout this repository - edit the `run_fax.sh` file to change run configurations and software paths - Run: "python MidwayBatch.py (output directory) (number of jobs) (partition: 0 [xenon1t], 1 [public], 2 [kicp])" + + + + +### Produce and Merge Datasets with `begin_production.py` +You can use this code to produce fax-only data with Qing's framework in one terminal command (after some editing). + +#### Setting Up +- Change the lines containing `username` and `file_header` in `sort_processed_files.sh`. `username` should be your midway username (to check squeue), and `file_header` is where all your produced data will appear (under a subdir defined by `process_name`) + +#### Running the Code +- This code imports `setup_production.py`, which creates a dictionary of options to be passed to fax/pax/hax through Qing's production scripts. +- You can hard-code all the options in `setup_production.py`, change the variable `interactive = 0`, then run `python begin_production.py`. +- Or you can just run `python begin_production.py`, and you will be prompted for all the options. + +#### Some confusing options: +- `process_name` : this will be the folder name under `file_header` given to your process +- `process_description` : this will be written to a text for your records +- `correlated` : this correlates s1 and s2 in time by adding dt, NOT in area +- `nodetype` : 0 for xenon1t, 1 for public, 2 for kicp +- `use_array_truth` : set to 1 to put truth information in arrays +- `minitree_type` : 0 for basics, 1 for S1S2Properties minitrees, 2 for PeakEfficiency minitrees diff --git a/montecarlo/fax_waveform/TruthSorting.py b/montecarlo/fax_waveform/TruthSorting.py index 59dcba8..b108651 100644 --- a/montecarlo/fax_waveform/TruthSorting.py +++ b/montecarlo/fax_waveform/TruthSorting.py @@ -29,8 +29,10 @@ TruthFile = sys.argv[1] OutputFile = sys.argv[2] -OutputFile = OutputFile.split('.')[0] - +if '.root' in OutputFile: + OutputFile = OutputFile.split('.root')[0] +else: + OutputFile = OutputFile.split('.pkl')[0] OutputFormat=0 if len(sys.argv)>3: OutputFormat = float(sys.argv[3]) diff --git a/montecarlo/fax_waveform/TruthSorting_arrays.py b/montecarlo/fax_waveform/TruthSorting_arrays.py index 863246f..0725346 100644 --- a/montecarlo/fax_waveform/TruthSorting_arrays.py +++ b/montecarlo/fax_waveform/TruthSorting_arrays.py @@ -28,7 +28,10 @@ TruthFile = sys.argv[1] OutputFile = sys.argv[2] -OutputFile = OutputFile.split('.')[0] +if '.root' in OutputFile: + OutputFile = OutputFile.split('.root')[0] +else: + OutputFile = OutputFile.split('.pkl')[0] OutputFormat=0 if len(sys.argv)>3: diff --git a/montecarlo/fax_waveform/begin_production.py b/montecarlo/fax_waveform/begin_production.py new file mode 100644 index 0000000..332fa2c --- /dev/null +++ b/montecarlo/fax_waveform/begin_production.py @@ -0,0 +1,68 @@ +import subprocess as subp +import sys, os +import time + +squeue_dict = {'0': 'xenon1t', '1': 'sandyb', '2': 'kicp'} +configs = {'0': 'basics_config', '1' : 's1s2_preserve_all', '2' : 'PeakEfficiency'} + +def wait_for_squeue(username, nodetype): + while len(subp.check_output(['squeue', '-u', username, '--partition', squeue_dict[nodetype]]).splitlines())>1: + print('waiting for squeue to free up, time = %i' % int(time.time())) + time.sleep(60) + +def make_processed_list(head_dirname, datetime, batchlist = 'processed_dataset_list.dat'): + dir_header = os.path.join(head_dirname, datetime) + pax_dir = os.path.join(dir_header, 'pax_%s/' % datetime) + print(pax_dir) + reduced_dir = os.path.join(dir_header, 'reduced_minitrees_%s/' % datetime) + batchlist_file = open(batchlist, 'w') + for subdir, dirs, files in os.walk(pax_dir): + for filename in files: + if '_pax.root' in filename: + batchlist_file.write(filename.split('.')[0] + '\n') + batchlist_file.close() + +def fax_produce(process, head_dirname, username): + dir_header = os.path.join(head_dirname, process['process_name']) + truth_dirname = os.path.join(dir_header, 'truth_minitrees_%s' % process['process_name']) + basics_dirname = os.path.join(dir_header, 'basics_minitrees_%s' % process['process_name']) + merged_dirname = os.path.join(dir_header, 'merged_minitrees_%s' % process['process_name']) + reduced_dirname = os.path.join(dir_header, 'reduced_minitrees_%s' % process['process_name']) + pax_dirname = os.path.join(dir_header, 'pax_%s' % process['process_name']) + production_commands = [] + production_commands.append('mkdir %s' % dir_header) + production_commands.append('echo "%s\n" >> %s' % (process['process_description'], os.path.join(dir_header, 'description.txt'))) + production_commands.append('echo $PWD >> %s' % os.path.join(dir_header, 'description.txt')) + production_commands.append('echo "\n%s" >> %s' % (str(process), os.path.join(dir_header, 'description.txt'))) + production_commands.append('cp run_fax.sh %s' % dir_header) + midway_batch_options = '%s %s %s %s %s %s %s %s %s %s %s' % (dir_header, process['nb_jobs'], + process['events_per_job'], process['pmt_afterpulse'], process['s2_afterpulse'], + process['photon_nb_low'], process['photon_nb_high'], process['electron_nb_low'], + process['electron_nb_high'], process['correlated'], process['nodetype']) + production_commands.append('python MidwayBatch.py %s >> %s' % (midway_batch_options, process['log_file'])) + production_commands.append('./sort_processed_files.sh %s >> %s' % (process['process_name'], process['log_file'])) + if process['minitree_type'] != '0': + batchlist = 'processed_dataset_list.dat' + production_commands.append('python BatchReduceDataSubmission.py %s %s %s %s %s 0 %s >> %s' % (batchlist, pax_dirname, + reduced_dirname, os.path.join(os.getcwd(), 'submission_reduce/'), process['nodetype'], process['minitree_type'], process['log_file'])) + production_commands.append('python BatchMergeTruthAndProcessed.py Configs/%s %s %s %s submission_merge/ %s 0 %s %s' % (configs[process['minitree_type']], + truth_dirname, reduced_dirname, merged_dirname, process['nodetype'], process['use_array_truth'], process['minitree_type'])) + else: + production_commands.append('python BatchMergeTruthAndProcessed.py Configs/%s %s %s %s submission_merge/ %s 0 %s %s' % (configs[process['minitree_type']], + truth_dirname, basics_dirname, merged_dirname, process['nodetype'], process['use_array_truth'], process['minitree_type'])) + production_commands.append('python MergePickles.py %s' % (merged_dirname)) + for command in production_commands[-2:]: + wait_for_squeue(username, process['nodetype']) + if process['minitree_type']!='0': + if 'BatchReduce' in command: + time.sleep(10) + make_processed_list(head_dirname, process['process_name'], batchlist=batchlist) + print('submitting command') + print(command) + subp.call(command, shell=True) + +import setup_production +process_list = setup_production.process_list +for process in process_list: + process['log_file'] = os.path.join('logs', process['log_file']) + fax_produce(process, setup_production.head_dirname, setup_production.username) diff --git a/montecarlo/fax_waveform/logs/170221_test.log b/montecarlo/fax_waveform/logs/170221_test.log new file mode 100644 index 0000000..c975e23 --- /dev/null +++ b/montecarlo/fax_waveform/logs/170221_test.log @@ -0,0 +1,153 @@ +1 +Submitted batch job 24206310 +/home/jh3226/testing_pax/montecarlo/fax_waveform +2 +Submitted batch job 24206311 +/home/jh3226/testing_pax/montecarlo/fax_waveform +3 +Submitted batch job 24206312 +/home/jh3226/testing_pax/montecarlo/fax_waveform +4 +Submitted batch job 24206313 +/home/jh3226/testing_pax/montecarlo/fax_waveform +5 +Submitted batch job 24206314 +/home/jh3226/testing_pax/montecarlo/fax_waveform +/home/jh3226/testing_pax/montecarlo/fax_waveform +Current job running number 1 +Current job running number 2 +Current job running number 3 +Current job running number 4 +Current job running number 5 +1 +Submitted batch job 24206319 +/home/jh3226/testing_pax/montecarlo/fax_waveform +2 +Submitted batch job 24206320 +/home/jh3226/testing_pax/montecarlo/fax_waveform +3 +Submitted batch job 24206322 +/home/jh3226/testing_pax/montecarlo/fax_waveform +4 +Submitted batch job 24206324 +/home/jh3226/testing_pax/montecarlo/fax_waveform +5 +Submitted batch job 24206325 +/home/jh3226/testing_pax/montecarlo/fax_waveform +/home/jh3226/testing_pax/montecarlo/fax_waveform +Current job running number 1 +Current job running number 2 +Current job running number 3 +Current job running number 4 +Current job running number 5 +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_truth.root to truth minitree dir +1 +Submitted batch job 24206376 +/home/jh3226/testing_pax/montecarlo/fax_waveform +2 +Submitted batch job 24206377 +/home/jh3226/testing_pax/montecarlo/fax_waveform +3 +Submitted batch job 24206378 +/home/jh3226/testing_pax/montecarlo/fax_waveform +4 +Submitted batch job 24206379 +/home/jh3226/testing_pax/montecarlo/fax_waveform +5 +Submitted batch job 24206380 +/home/jh3226/testing_pax/montecarlo/fax_waveform +To process -> FakeWaveform_XENON1T_000000_pax +Current job running number 1 +To process -> FakeWaveform_XENON1T_000003_pax +Current job running number 2 +To process -> FakeWaveform_XENON1T_000004_pax +Current job running number 3 +To process -> FakeWaveform_XENON1T_000001_pax +Current job running number 4 +To process -> FakeWaveform_XENON1T_000002_pax +Current job running number 5 +1 +Submitted batch job 24206599 +/home/jh3226/testing_pax/montecarlo/fax_waveform +2 +Submitted batch job 24206600 +/home/jh3226/testing_pax/montecarlo/fax_waveform +3 +Submitted batch job 24206601 +/home/jh3226/testing_pax/montecarlo/fax_waveform +4 +Submitted batch job 24206602 +/home/jh3226/testing_pax/montecarlo/fax_waveform +5 +Submitted batch job 24206604 +/home/jh3226/testing_pax/montecarlo/fax_waveform +/home/jh3226/testing_pax/montecarlo/fax_waveform +Current job running number 1 +Current job running number 2 +Current job running number 3 +Current job running number 4 +Current job running number 5 +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_truth.root to truth minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Basics.root to basics minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Fundamentals.root to processed minitree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax.root to pax tree dir +moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_truth.root to truth minitree dir +1 +Submitted batch job 24206622 +/home/jh3226/testing_pax/montecarlo/fax_waveform +2 +Submitted batch job 24206623 +/home/jh3226/testing_pax/montecarlo/fax_waveform +3 +Submitted batch job 24206624 +/home/jh3226/testing_pax/montecarlo/fax_waveform +4 +Submitted batch job 24206625 +/home/jh3226/testing_pax/montecarlo/fax_waveform +5 +Submitted batch job 24206626 +/home/jh3226/testing_pax/montecarlo/fax_waveform +To process -> FakeWaveform_XENON1T_000000_pax +Current job running number 1 +To process -> FakeWaveform_XENON1T_000003_pax +Current job running number 2 +To process -> FakeWaveform_XENON1T_000004_pax +Current job running number 3 +To process -> FakeWaveform_XENON1T_000001_pax +Current job running number 4 +To process -> FakeWaveform_XENON1T_000002_pax +Current job running number 5 diff --git a/montecarlo/fax_waveform/processed_dataset_list.dat b/montecarlo/fax_waveform/processed_dataset_list.dat new file mode 100644 index 0000000..e1a5f6e --- /dev/null +++ b/montecarlo/fax_waveform/processed_dataset_list.dat @@ -0,0 +1,5 @@ +FakeWaveform_XENON1T_000000_pax +FakeWaveform_XENON1T_000003_pax +FakeWaveform_XENON1T_000004_pax +FakeWaveform_XENON1T_000001_pax +FakeWaveform_XENON1T_000002_pax diff --git a/montecarlo/fax_waveform/reduce_peak_level.py b/montecarlo/fax_waveform/reduce_peak_level.py new file mode 100644 index 0000000..565a03a --- /dev/null +++ b/montecarlo/fax_waveform/reduce_peak_level.py @@ -0,0 +1,53 @@ +import numpy as np +import hax +import sys + +#truth_filename = '/project/lgrandi/jhowlett/170117_1620/truth_minitrees_170117_1620/FakeWaveform_XENON1T_000000_truth' +#processed_filename = '/project/lgrandi/jhowlett/170117_1620/000000/FakeWaveform_XENON1T_000000_pax.root' +#processed_filename = 'FakeWaveform_XENON1T_000000_pax' + + +if len(sys.argv)<2: + print("========== Syntax ===========") + print("python reduce_peak_level.py ") + exit() + +#with open('datasets.dat', 'r') as dataset_file: +# datasets = dataset_file.readlines() + +#datasets = ['FakeWaveform_XENON1T_000000_pax', 'FakeWaveform_XENON1T_000001_pax'] + + + +#hax.init(experiment='XENON1T', main_data_paths=['/project/lgrandi/jhowlett/170117_1620/pax_170117_1620/'], minitree_paths = ['temp_minitrees'], pax_version_policy='loose') + +class PeakEfficiency(hax.minitrees.TreeMaker): + __version__ = '0.0.1' + uses_arrays = True + extra_branches = ['peaks.*'] + + def extract_data(self, event): + peak_fields = ['area', 'hit_time_std', 'hit_time_mean', 'area_fraction_top', 'height', 'n_contributing_channels'] + decile = [5, 7, 9] + + result = {} + result['event_number'] = event.event_number + result['index'] = event.event_number + type_ints = {'s1': 1, 's2': 2, 'unknown': 3} + peaks = [peak for peak in event.peaks if ((peak.type != 'lone_hit') and (peak.detector == 'tpc'))] + for peak_field in peak_fields: + result[peak_field] = np.array([getattr(peaks[i], peak_field) for i in range(len(peaks))]) + result['type'] = np.array([type_ints[getattr(peak, 'type')] for peak in peaks]) + for dec in decile: + result['range_%i0p_area' % dec] = np.array([getattr(peaks[i], 'range_area_decile')[dec] for i in range(len(peaks))]) + return result + +dataset = sys.argv[1] +datapath = sys.argv[2] +print("======= To be reduced: "+dataset) +print(datapath) +hax.init(experiment='XENON1T', main_data_paths=[datapath], use_rundb_locations=False, pax_version_policy='loose')# changed @2016-07-06, for the data after 07-03 +#hax.init(main_data_paths=[datapath])# changed @2016-07-06, for the data after 07-03 +#print(hax.config['main_data_paths']) + +data2 = hax.minitrees.load(dataset, treemakers=[PeakEfficiency], force_reload=True) diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index 8183a53..18d6420 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -43,6 +43,7 @@ SUBRUN=$9 # Setup the software CVMFSDIR=/cvmfs/xenon.opensciencegrid.org export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" +#export PATH="/project/lgrandi/anaconda3/bin:$PATH" source activate pax_${PAXVERSION} &> /dev/null # Use path of this script for Python scripts below diff --git a/montecarlo/fax_waveform/setup_production.py b/montecarlo/fax_waveform/setup_production.py new file mode 100644 index 0000000..92d8eb2 --- /dev/null +++ b/montecarlo/fax_waveform/setup_production.py @@ -0,0 +1,60 @@ +import sys +### Edit these three lines ### +with open('sort_processed_files.sh') as bashfile: + lines = bashfile.readlines() +head_dirname = lines[1].split('=')[1].split()[0] # all data will go under here (under subdir) +username = lines[2].split('=')[1].split()[0] # midway username +interactive = 1 # 1 for terminal prompt options, 0 for hardcoded options below + +process_list = [] + +fields = [ + 'process_name', 'process_description', 'log_file', 'nb_jobs', + 'events_per_job', 'pmt_afterpulse', 's2_afterpulse', + 'photon_nb_low', 'photon_nb_high', 'electron_nb_low', + 'electron_nb_high', 'correlated', 'nodetype', 'minitree_type', + 'use_array_truth' + ] + +process = {} +process['process_name'] = '170222_1108' +process['process_description'] = 'testing on new branch' +process['log_file'] = '170221_test.log' +process['nb_jobs'] = '5' +process['events_per_job'] = '2' +process['pmt_afterpulse'] = '1' +process['s2_afterpulse'] = '1' +process['photon_nb_low'] = '0' +process['photon_nb_high'] = '1000' +process['electron_nb_low'] = '0' +process['electron_nb_high'] = '100' +process['correlated'] = '1' +process['nodetype'] = '0' +process['minitree_type'] = '1' +process['use_array_truth'] = '1' +if interactive == 0: + process_list.append(process) + +def setup_process(fields, process_nb): + process = {} + for field in fields: + process[field] = input('set %s for process %i >\t' % (field, process_nb)) + return process + +if interactive: + nb_processes = int(input('How many processes? >\t')) + + for i in range(1, nb_processes+1): + process_list.append(setup_process(fields, i)) + + print('\nConfirm Process List:') + for (i, process) in enumerate(process_list): + print('\nProcess %i:' % (i+1)) + for field in fields: + print('%s:\t%s' % (field, process[field])) + confirmation = input('\nConfirm (y/n) > ') + if confirmation=='n': + print('Edit Process %i:' % (i+1)) + process_list[i] = setup_process(fields, i+1) + + diff --git a/montecarlo/fax_waveform/sort_processed_files.sh b/montecarlo/fax_waveform/sort_processed_files.sh new file mode 100755 index 0000000..0624c09 --- /dev/null +++ b/montecarlo/fax_waveform/sort_processed_files.sh @@ -0,0 +1,43 @@ +#!/bin/bash +file_header=/project/lgrandi/jhowlett/ # change this line +username=jh3226 # and this line +datetime=$1 +truth_dir=$file_header"$datetime/truth_minitrees_"$datetime +processed_dir=$file_header"$datetime/processed_minitrees_"$datetime +merged_dir=$file_header"$datetime/merged_minitrees_"$datetime +pax_dir=$file_header"$datetime/pax_"$datetime +reduced_dir=$file_header"$datetime/reduced_minitrees_"$datetime +basics_dir=$file_header"$datetime/basics_minitrees_"$datetime + +#if [ -d "$truth_dir" ] || [ -d "$processed_dir" ]; then +# echo "minitree dirs already exist for $datetime; exiting..." +# exit +#fi +mkdir $truth_dir +mkdir $processed_dir +mkdir $merged_dir +mkdir $pax_dir +mkdir $reduced_dir +mkdir $basics_dir +for dirname in $file_header$datetime"/0*"; do + for filename in $dirname/*; do + if [[ $filename =~ .*truth\.root$ ]]; then + echo "moving $filename to truth minitree dir" + mv $filename $truth_dir/ + elif [[ $filename =~ .*pax_Basics\.root$ ]]; then + echo "moving $filename to basics minitree dir" + mv $filename $basics_dir/ + elif [[ $filename =~ .*pax_.*\.root$ ]]; then + echo "moving $filename to processed minitree dir" + mv $filename $processed_dir/ + elif [[ $filename =~ .*pax\.root$ ]]; then + echo "moving $filename to pax tree dir" + mv $filename $pax_dir/ + fi + done +done +if [[ $2 =~ kill ]]; then + for dirname in $file_header$datetime"/0*"; do + rm -rf $dirname + done +fi diff --git a/montecarlo/fax_waveform/submission_merge/desc.txt b/montecarlo/fax_waveform/submission_merge/desc.txt new file mode 100644 index 0000000..7a08d23 --- /dev/null +++ b/montecarlo/fax_waveform/submission_merge/desc.txt @@ -0,0 +1 @@ +placeholder, this is where merging files/errors will be diff --git a/montecarlo/fax_waveform/submission_reduce/desc.txt b/montecarlo/fax_waveform/submission_reduce/desc.txt new file mode 100644 index 0000000..8ce0c61 --- /dev/null +++ b/montecarlo/fax_waveform/submission_reduce/desc.txt @@ -0,0 +1,2 @@ +placeholder to load to git: +this folder holds all the temporary reduced minitrees From 6748187f6a12d2e056d364d02d4955ede1b5ad27 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Wed, 22 Feb 2017 13:59:59 -0600 Subject: [PATCH 12/21] New branch for S1-S2 area correlated simulation. --- .../CreateFakeCSV_CorrelatedS1S2.py | 122 ++++++++++++++++++ .../fax_waveform/MakePickleFromHistogram.py | 56 ++++++++ .../fax_waveform/PhotonChargeGenerator.py | 101 +++++++++++++++ montecarlo/fax_waveform/run_fax.sh | 13 +- 4 files changed, 289 insertions(+), 3 deletions(-) create mode 100644 montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py create mode 100644 montecarlo/fax_waveform/MakePickleFromHistogram.py create mode 100644 montecarlo/fax_waveform/PhotonChargeGenerator.py diff --git a/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py b/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py new file mode 100644 index 0000000..15ec43a --- /dev/null +++ b/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py @@ -0,0 +1,122 @@ +################################# +## Sub-code used in WF simulation +## It creates a csv file for the input of fax +## using the 2d-pdf from an input histogram +## by Qing Lin +## @ 2016-09-12 +## +## HARDCODE WARNING: The FV dimensions below need to be modified +## according to the detector you wish to simulate +################################# +import sys +import numpy as np +import scipy as sp + +if len(sys.argv)<2: + print("========= Syntax ==========") + print("python CreateFakeCSV.py ..... ") + print("") + print("") + print("") + print("") + print("") + print("") + print("") + exit() + +Detector = sys.argv[1] +Band2DPDFFilename = sys.argv[2] +NominalG1Value = float(sys.argv[3]) +NominalG2Value = float(sys.argv[4]) +NumEvents = int(sys.argv[5]) +DefaultType = sys.argv[6] +OutputFilename = sys.argv[7] + +#################################### +## Some nuisance parameters (HARDCODE WARNING): +#################################### +MaxDriftTime = 650. # us + + +#################################### +## Some functions (HARDCODE WARNING): +#################################### + +# Current FV cut for Xe1T +scalecmtomm=1 +def radius2_cut(zpos): + return 1400*scalecmtomm**2+(zpos+100*scalecmtomm)*(2250-1900)*scalecmtomm/100 + +def IfPassFV(x,y,z): + + if Detector == "XENON100": + # check if the x,y,z passing X48kg0 + I = np.power( (z+15.)/14.6, 4.) + I += np.power( (x**2+y**2)/20000., 4.) + if I<1: + return True + elif Detector == "XENON1T": # NEED TO UPDATE THIS + Zlower, Zupper = -90*scalecmtomm, -15*scalecmtomm + Zcut = ((z>=Zlower) & (z<=Zupper)) + R2upper=radius2_cut(z) + Rcut = (x**2+y**2") + print("") + print("") + exit() + +InputROOTFilename = sys.argv[1] +HistogramName = sys.argv[2] +OutputPickleFilename = sys.argv[3] + + +#################################### +# Open the root file and load the TH2D +#################################### +pfile = TFile(InputROOTFilename) +hist = pfile.Get(HistogramName) + +#################################### +# translate into a dictionary +#################################### +data = {} +data['s1nbins'] = int( hist.GetXaxis().GetNbins() ) +data['s1lower'] = float( hist.GetXaxis().GetXmin() ) +data['s1upper'] = float( hist.GetXaxis().GetXmax() ) +data['lognbins'] = int( hist.GetYaxis().GetNbins() ) +data['loglower'] = float( hist.GetYaxis().GetXmin() ) +data['logupper'] = float( hist.GetYaxis().GetXmax() ) +data['map'] = [] + +for i in range(data['s1nbins']): + TmpList = [] + for j in range(data['lognbins']): + cont = float(hist.GetBinContent(i+1, j+1) ) + TmpList.append(cont) + data['map'].append( list(TmpList) ) + +#################################### +## output to pickle +#################################### + +pickle.dump( data, open(OutputPickleFilename, 'wb') ) diff --git a/montecarlo/fax_waveform/PhotonChargeGenerator.py b/montecarlo/fax_waveform/PhotonChargeGenerator.py new file mode 100644 index 0000000..e98b643 --- /dev/null +++ b/montecarlo/fax_waveform/PhotonChargeGenerator.py @@ -0,0 +1,101 @@ +#################################### +## This code generate number of photons and charges +## according to a 2-D pdf input +## by Qing Lin +## Created @ 2017-02-21 +#################################### + +import sys, os +import pickle +import numpy as np +import scipy as sp +from scipy.interpolate import interp1d + +class MyPhotonChargeGenerator: + def __init__(self, Input2DFilename, g1, g2): + self.m_pG1Value = g1 + self.m_pG2Value = g2 + self.Load(Input2DFilename) + return + + def __eq__(self, other): + self.m_pG1Value = other.m_pG1Value + self.m_pG2Value = other.m_pG2Value + # range of S1 and log + self.m_pS1Nbins = other.m_pS1Nbins + self.m_pS1Lower = other.m_pS1Lower + self.m_pS1Upper = other.m_pS1Upper + self.m_pS1Step = other.m_pS1Step + self.m_pLogNbins = other.m_pLogNbins + self.m_pLogLower = other.m_pLogLower + self.m_pLogUpper = other.m_pLogUpper + self.m_pLogStep = other.m_pLogStep + # interpolators + self.m_pS1Interpolator = other.m_pS1Interpolator + self.m_pLogInterpolators = other.m_pLogInterpolators + return + + + def Load(self, Input2DFilename): + data = pickle.load( open(Input2DFilename, 'rb') ) + if not self.CheckIfComplete(data): + raise ValueError("Pickle file not complete") + self.m_pS1Nbins = data['s1nbins'] + self.m_pS1Lower = data['s1lower'] + self.m_pS1Upper = data['s1upper'] + self.m_pS1Step = (self.m_pS1Upper - self.m_pS1Lower) / float( self.m_pS1Nbins) + self.m_pLogNbins = data['lognbins'] + self.m_pLogLower = data['loglower'] + self.m_pLogUpper = data['logupper'] + self.m_pLogStep = (self.m_pLogUpper - self.m_pLogLower) / float( self.m_pLogNbins) + # load S1 pdf and calculate cdf + S1s = np.linspace(self.m_pS1Lower, self.m_pS1Upper, self.m_pS1Nbins+1) + S1s = S1s[1:] + S1PDFs = [ np.sum(A) for A in data['map'] ] + self.m_pS1Interpolator = interp1d(np.cumsum(S1PDFs)/np.sum(S1PDFs), S1s, bounds_error=False, fill_value=(0, 1)) + # load S2 pdfs and calculate cdf + Logs = np.linspace(self.m_pLogLower, self.m_pLogUpper, self.m_pLogNbins+1) + Logs = Logs[1:] + self.m_pLogInterpolators = [] + for LogPDFs in data['map']: + self.m_pLogInterpolators.append( + interp1d(np.cumsum(LogPDFs)/np.sum(LogPDFs), Logs, bounds_error=False, fill_value=(0,1) ) + ) + return + + + def CheckIfComplete(self, Data): + Items = [ + 's1nbins', + 's1lower', + 's1upper', + 'lognbins', + 'loglower', + 'logupper', + 'map', + ] + for item in Items: + if item not in Data: + return False + return True + + def GetPhotonChargeNum(self): + s1_cdf = np.random.uniform(0, 1) + s1 = self.m_pS1Interpolator(s1_cdf) + s1_bin = int( (s1 - self.m_pS1Lower) / self.m_pS1Step ) + s1_bin_upper = s1_bin + 1 + if s1_bin_upper>=len(self.m_pLogInterpolators): + s1_bin_upper = s1_bin + s1lower = float(s1_bin)*self.m_pS1Step + self.m_pS1Lower + s1upper = float(s1_bin_upper)*self.m_pS1Step + self.m_pS1Lower + log_cdf = np.random.uniform(0, 1) + loglower = self.m_pLogInterpolators[s1_bin](log_cdf) + logupper = self.m_pLogInterpolators[s1_bin_upper](log_cdf) + log = loglower + if s1lower!=s1upper: + log = (s1 - s1lower) / (s1upper - s1lower) * (logupper - loglower) + loglower + s2 = np.power(10., log)*s1 + return (s1/self.m_pG1Value, s2/self.m_pG2Value) + + + diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index 18d6420..dbcf6a8 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -30,7 +30,8 @@ PMTAfterpulseEnableFlag=$5 S2AfterpulseEnableFlag=$6 # Select fax+pax version -PAXVERSION=head +#~ PAXVERSION=head +PAXVERSION=v6.5.0 # temporary change # Specify number of events NumEvents=$7 @@ -42,8 +43,8 @@ SUBRUN=$9 # Setup the software CVMFSDIR=/cvmfs/xenon.opensciencegrid.org -export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" -#export PATH="/project/lgrandi/anaconda3/bin:$PATH" +#export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" +export PATH="/project/lgrandi/anaconda3/bin:$PATH" source activate pax_${PAXVERSION} &> /dev/null # Use path of this script for Python scripts below @@ -78,6 +79,7 @@ FAX_FILENAME=${FILENAME}_truth # fax truth info PKL_FILENAME=${FILENAME}_truth.pkl # converted fax truth info RAW_FILENAME=${FILENAME}_raw # fax simulated raw data PAX_FILENAME=${FILENAME}_pax # pax processed data +PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path HAX_FILENAME=${FILENAME}_hax # hax reduced data CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini @@ -124,6 +126,11 @@ HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals'] (time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log + +# custom minitree +(time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log + + # Cleanup rm -f pax* From 72f58c7469344db129d290da8b37fb02f2e4fac3 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Wed, 22 Feb 2017 14:32:17 -0600 Subject: [PATCH 13/21] Added the batch mode for the area correlated S1&S2 mode --- .../MidwayBatch_AreaCorrelatedS1S2.py | 101 +++++++++++++ .../run_fax_AreaCorrelatedS1S2.py | 137 ++++++++++++++++++ 2 files changed, 238 insertions(+) create mode 100644 montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py create mode 100644 montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.py diff --git a/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py new file mode 100644 index 0000000..c93e465 --- /dev/null +++ b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py @@ -0,0 +1,101 @@ +#################################### +## batch code for WF simulation +#################################### +import sys, array, os, getpass +from subprocess import call +import subprocess as subp +import time +import math as math +from subprocess import Popen, PIPE + +if len(sys.argv)<2: + print("========= Syntax ========") + print("python MidwayBatch_AreaCorrelatedS1S2.py ....") + print("") + print("") + print("") + print("") + print("") + print("<2D band for S1-S2 area correlation (abs.)>") + print("") + print("") + print("") + exit() + +OutputGeneralPath = sys.argv[1] +NumJobs = int(sys.argv[2]) +NumEvents = int(sys.argv[3]) +PMTAfterpulseFlag = int(sys.argv[4]) +S2AfterpulseFlag = int(sys.argv[5]) +Input2DBandFile = sys.argv[6] +Nomial_g1 = float(sys.argv[7]) +Nomial_g2 = float(sys.argv[8]) +IfUsePublicNodes = int(sys.argv[9]) + +MaxNumJob = 64 +if not IfUsePublicNodes: + MaxNumJob=200 + + +##### Start batching ######### +CurrentPath = os.getcwd() +print (CurrentPath) +CurrentUser = getpass.getuser() +for i in range(NumJobs): + + RunString = "%06d" % i + + # create folder + OutputPath = OutputGeneralPath + "/" + RunString + if os.path.exists(OutputPath): + subp.call("rm -r "+OutputPath, shell=True) + subp.call("mkdir -p "+OutputPath, shell=True) + + # define filenames + SubmitFile = OutputPath+"/submit_"+ RunString + ".sh" + SubmitOutputFilename = OutputPath+"/submit_"+ RunString + ".log" + SubmitErrorFilename = OutputPath+"/submit_"+ RunString + ".log" + + # create the basic submit + subp.call("echo '#!/bin/bash\n' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --output="+SubmitOutputFilename+"' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --error="+SubmitErrorFilename+"' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --time=03:59:00' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --account=pi-lgrandi' >> "+SubmitFile, shell=True) + if IfUsePublicNodes==0: + subp.call("echo '#SBATCH --qos=xenon1t' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --partition=xenon1t\n' >> "+SubmitFile, shell=True) + elif IfUsePublicNodes==2: + subp.call("echo '#SBATCH --qos=xenon1t-kicp' >> "+SubmitFile, shell=True) + subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True) + + Command = CurrentPath+"/./run_fax_AreaCorrelatedS1S2.sh "+Input2DBandFile+""+str(Nomial_g1)+" "+str(Nomial_g2)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString) + subp.call("echo '"+Command+"\n' >> "+SubmitFile, shell=True) + + SubmitPath = OutputPath + + #submit + IfSubmitted=0 + while IfSubmitted==0: + Partition = "sandyb" # public + if not IfUsePublicNodes: + Partition = "xenon1t" + elif IfUsePublicNodes==2: + Partition = "kicp" + p1 = Popen(["squeue","--partition="+Partition, "--user="+CurrentUser], stdout=PIPE) + p2 = Popen(["wc", "-l"], stdin=p1.stdout, stdout=PIPE) + p1.stdout.close() # Allow p1 to receive a SIGPIPE if p2 exits. + output = p2.communicate()[0] + Status=subp.call("squeue --partition="+Partition+" --user="+CurrentUser +" | wc -l", shell=True) + Output=int(output) + #print(Status) + + print("Current job running number "+str(Output)) + + if Status==0 and Output +# +############################################ + +echo "Start time: " `/bin/date` +echo "Job is running on node: " `/bin/hostname` +echo "Job running as user: " `/usr/bin/id` +echo "Job is running in directory: $PWD" + +###### General parameters ##### +Detector=XENON1T + +###### Simulation parameters ##### +Input2DBandFile=$1 +Nomial_g1=$2 +Nomial_g2=$3 + +RecoilType=ER + +echo 'IfS1S2Correlation = ${IfS1S2Correlation}' +# enable s2 after pulse depending on the argument +# 1 for enable +PMTAfterpulseEnableFlag=$4 +S2AfterpulseEnableFlag=$5 + +# Select fax+pax version +#~ PAXVERSION=head +PAXVERSION=v6.5.0 # temporary change + +# Specify number of events +NumEvents=$6 + +# This run number (from command line argument) +SUBRUN=$8 + +######################################## + +# Setup the software +CVMFSDIR=/cvmfs/xenon.opensciencegrid.org +#export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" +export PATH="/project/lgrandi/anaconda3/bin:$PATH" +source activate pax_${PAXVERSION} &> /dev/null + +# Use path of this script for Python scripts below +# (In case user modified them) +#MY_PATH=`dirname \"$0\"` +MY_PATH=$(cd `dirname $0`; pwd) +echo $MY_PATH +RELEASEDIR=`( cd "$MY_PATH" && pwd )` + +# Setting up directories +#start_dir=$PWD + + +OUTDIR=$7/${SUBRUN} +mkdir -p ${OUTDIR} +cd ${OUTDIR} + +#if [ "$OSG_WN_TMP" == "" ]; +#then +# OSG_WN_TMP=$PWD +#fi +#cd $OSG_WN_TMP +# +#work_dir=`mktemp -d --tmpdir=$OSG_WN_TMP` +#cd $work_dir + +# Filenaming +FILEROOT=FakeWaveform_${Detector}_${SUBRUN} +FILENAME=${OUTDIR}/${FILEROOT} +CSV_FILENAME=${FILENAME}.csv # Fake input data +FAX_FILENAME=${FILENAME}_truth # fax truth info +PKL_FILENAME=${FILENAME}_truth.pkl # converted fax truth info +RAW_FILENAME=${FILENAME}_raw # fax simulated raw data +PAX_FILENAME=${FILENAME}_pax # pax processed data +PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path +HAX_FILENAME=${FILENAME}_hax # hax reduced data +CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini +NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini +echo ${CustomIniFilename} + + +# Create the fake input data +python ${RELEASEDIR}/CreateFakeCSV_CorrelatedS1S2.py ${Detector} ${Input2DBandFile} ${Nomial_g1} ${Nomial_g2} ${NumEvents} ${RecoilType} ${CSV_FILENAME} + +# Start of simulations # + +# fax stage +if (($S2AfterpulseEnableFlag==0)); then + if (($PMTAfterpulseEnableFlag==0)); then + echo 'Both S2 and PMT afterpulse disabled' + (time paxer --input ${CSV_FILENAME} --config ${Detector} reduce_raw_data Simulation --config_path ${NoPMTAfterpulseIniFilename} ${CustomIniFilename} --config_string "[WaveformSimulator]truth_file_name=\"${FAX_FILENAME}\"" --output ${RAW_FILENAME};) &> ${RAW_FILENAME}.log + else + echo 'Only S2 afterpulse disabled' + (time paxer --input ${CSV_FILENAME} --config ${Detector} reduce_raw_data Simulation --config_path ${CustomIniFilename} --config_string "[WaveformSimulator]truth_file_name=\"${FAX_FILENAME}\"" --output ${RAW_FILENAME};) &> ${RAW_FILENAME}.log + fi +else + if (($PMTAfterpulseEnableFlag==0)); then + echo 'Only PMT afterpulse disabled' + (time paxer --input ${CSV_FILENAME} --config ${Detector} reduce_raw_data Simulation --config_path ${NoPMTAfterpulseIniFilename} --config_string "[WaveformSimulator]truth_file_name=\"${FAX_FILENAME}\"" --output ${RAW_FILENAME};) &> ${RAW_FILENAME}.log + else + echo 'Both S2 and PMT afterpulse enabled' + (time paxer --input ${CSV_FILENAME} --config ${Detector} reduce_raw_data Simulation --config_string "[WaveformSimulator]truth_file_name=\"${FAX_FILENAME}\"" --output ${RAW_FILENAME};) &> ${RAW_FILENAME}.log + fi +fi + +# (time paxer --input ${CSV_FILENAME} --config ${Detector} reduce_raw_data Simulation --config_string "[WaveformSimulator]truth_file_name=\"${FAX_FILENAME}\"" --output ${RAW_FILENAME};) &> ${RAW_FILENAME}.log + + +# convert fax truth to pickle +python ${RELEASEDIR}/ConvertFaxTruthToPickle.py ${FAX_FILENAME} ${PKL_FILENAME} + +# pax stage +(time paxer --ignore_rundb --input ${RAW_FILENAME} --config ${Detector} --output ${PAX_FILENAME};) &> ${PAX_FILENAME}.log + +# hax stage +HAXPYTHON="import hax; " +HAXPYTHON+="hax.init(main_data_paths=['${OUTDIR}'], minitree_paths=['${OUTDIR}'], pax_version_policy = 'loose'); " +HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals']);" + +(time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log + + +# custom minitree +(time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log + + +# Cleanup +rm -f pax* + + +#cd $start_dir +#rm -fr $work_dir From 2f93b0898d6d6592ad16427ed60e00dda38f0181 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Wed, 22 Feb 2017 14:34:29 -0600 Subject: [PATCH 14/21] Added the batch mode for the area correlated S1&S2 mode --- montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py index c93e465..e8b4d34 100644 --- a/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py +++ b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py @@ -69,7 +69,7 @@ subp.call("echo '#SBATCH --qos=xenon1t-kicp' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True) - Command = CurrentPath+"/./run_fax_AreaCorrelatedS1S2.sh "+Input2DBandFile+""+str(Nomial_g1)+" "+str(Nomial_g2)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString) + Command = CurrentPath+"/./run_fax_AreaCorrelatedS1S2.sh "+Input2DBandFile+""+str(Nomial_g1)+" "+str(Nomial_g2)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString subp.call("echo '"+Command+"\n' >> "+SubmitFile, shell=True) SubmitPath = OutputPath From 26acac0edae434f06030763a9aa0ec38bdf29b30 Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Wed, 22 Feb 2017 16:53:04 -0600 Subject: [PATCH 15/21] Put also the merging process in run_fax*. Note this is using the minitree S1S2Properties --- montecarlo/fax_waveform/run_fax.sh | 6 ++++++ ..._AreaCorrelatedS1S2.py => run_fax_AreaCorrelatedS1S2.sh} | 6 ++++++ 2 files changed, 12 insertions(+) rename montecarlo/fax_waveform/{run_fax_AreaCorrelatedS1S2.py => run_fax_AreaCorrelatedS1S2.sh} (93%) diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index dbcf6a8..90ae165 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -80,6 +80,8 @@ PKL_FILENAME=${FILENAME}_truth.pkl # converted fax truth info RAW_FILENAME=${FILENAME}_raw # fax simulated raw data PAX_FILENAME=${FILENAME}_pax # pax processed data PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path +MERGEDTRUTH_FILENAME=${FILENAME}_merged_truth +MERGED_FILENAME=${FILENAME}_merged HAX_FILENAME=${FILENAME}_hax # hax reduced data CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini @@ -130,6 +132,10 @@ HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals'] # custom minitree (time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log +# merge +python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl +python ${RELEASEDIR}/MergeTruthAndProcessed.py ${MERGEDTRUTH_FILENAME}.pkl ${RELEASEDIR}/Configs/QingConfig ${MERGED_FILENAME}.pkl + # Cleanup rm -f pax* diff --git a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.py b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh similarity index 93% rename from montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.py rename to montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh index d7595f9..cf8439c 100644 --- a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.py +++ b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh @@ -78,6 +78,8 @@ RAW_FILENAME=${FILENAME}_raw # fax simulated raw data PAX_FILENAME=${FILENAME}_pax # pax processed data PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path +MERGEDTRUTH_FILENAME=${FILENAME}_merged_truth +MERGED_FILENAME=${FILENAME}_merged HAX_FILENAME=${FILENAME}_hax # hax reduced data CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini @@ -128,6 +130,10 @@ # custom minitree (time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log +# merge +python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl +python ${RELEASEDIR}/MergeTruthAndProcessed.py ${MERGEDTRUTH_FILENAME}.pkl ${RELEASEDIR}/Configs/QingConfig ${MERGED_FILENAME}.pkl + # Cleanup rm -f pax* From 50a6a5784ccdcc1abe8edb5c15f4200ee017f99a Mon Sep 17 00:00:00 2001 From: Qing Lin Date: Wed, 22 Feb 2017 17:59:15 -0600 Subject: [PATCH 16/21] Debugged --- .../CreateFakeCSV_CorrelatedS1S2.py | 4 ++-- .../MidwayBatch_AreaCorrelatedS1S2.py | 2 +- montecarlo/fax_waveform/ReduceDataNormal.py | 2 +- montecarlo/fax_waveform/run_fax.sh | 15 ++++++++------- .../fax_waveform/run_fax_AreaCorrelatedS1S2.sh | 18 ++++++++---------- 5 files changed, 20 insertions(+), 21 deletions(-) diff --git a/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py b/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py index 15ec43a..2a476f9 100644 --- a/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py +++ b/montecarlo/fax_waveform/CreateFakeCSV_CorrelatedS1S2.py @@ -116,7 +116,7 @@ def RandomizeFV(): fout.write(str(Y)+",") fout.write(str(-Z)+",") NumPhoton, NumElectron = pGen.GetPhotonChargeNum() - fout.write(str(NumPhoton)+",") - fout.write(str(NumElectron)+",") + fout.write(str(int(NumPhoton))+",") + fout.write(str(int(NumElectron))+",") fout.write(str(DefaultEventTime)+"\n") diff --git a/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py index e8b4d34..144ddb2 100644 --- a/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py +++ b/montecarlo/fax_waveform/MidwayBatch_AreaCorrelatedS1S2.py @@ -69,7 +69,7 @@ subp.call("echo '#SBATCH --qos=xenon1t-kicp' >> "+SubmitFile, shell=True) subp.call("echo '#SBATCH --partition=kicp\n' >> "+SubmitFile, shell=True) - Command = CurrentPath+"/./run_fax_AreaCorrelatedS1S2.sh "+Input2DBandFile+""+str(Nomial_g1)+" "+str(Nomial_g2)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString + Command = CurrentPath+"/./run_fax_AreaCorrelatedS1S2.sh "+Input2DBandFile+" "+str(Nomial_g1)+" "+str(Nomial_g2)+" "+str(PMTAfterpulseFlag)+" "+str(S2AfterpulseFlag)+" "+str(NumEvents)+" "+OutputGeneralPath+" "+RunString subp.call("echo '"+Command+"\n' >> "+SubmitFile, shell=True) SubmitPath = OutputPath diff --git a/montecarlo/fax_waveform/ReduceDataNormal.py b/montecarlo/fax_waveform/ReduceDataNormal.py index 47f87a9..c98b382 100644 --- a/montecarlo/fax_waveform/ReduceDataNormal.py +++ b/montecarlo/fax_waveform/ReduceDataNormal.py @@ -52,7 +52,7 @@ # hax.init(experiment='XENON1T') # my own builder -from collections import defaultdict +#~ from collections import defaultdict class S1S2Properties(hax.minitrees.TreeMaker): """Computing properties of the S1 diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index 90ae165..36bfca3 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -83,6 +83,7 @@ PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path MERGEDTRUTH_FILENAME=${FILENAME}_merged_truth MERGED_FILENAME=${FILENAME}_merged HAX_FILENAME=${FILENAME}_hax # hax reduced data +MINITREE_FILENAME=${FILENAME}_pax_S1S2Properties.root CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini echo ${CustomIniFilename} @@ -121,20 +122,20 @@ python ${RELEASEDIR}/ConvertFaxTruthToPickle.py ${FAX_FILENAME} ${PKL_FILENAME} # pax stage (time paxer --ignore_rundb --input ${RAW_FILENAME} --config ${Detector} --output ${PAX_FILENAME};) &> ${PAX_FILENAME}.log -# hax stage -HAXPYTHON="import hax; " -HAXPYTHON+="hax.init(main_data_paths=['${OUTDIR}'], minitree_paths=['${OUTDIR}'], pax_version_policy = 'loose'); " -HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals']);" +#~ # hax stage +#~ HAXPYTHON="import hax; " +#~ HAXPYTHON+="hax.init(main_data_paths=['${OUTDIR}'], minitree_paths=['${OUTDIR}'], pax_version_policy = 'loose'); " +#~ HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals']);" -(time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log +#~ (time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log # custom minitree (time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log # merge -python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl -python ${RELEASEDIR}/MergeTruthAndProcessed.py ${MERGEDTRUTH_FILENAME}.pkl ${RELEASEDIR}/Configs/QingConfig ${MERGED_FILENAME}.pkl +(time python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl;) &> ${MERGEDTRUTH_FILENAME}.log +(time python ${RELEASEDIR}/MergeTruthAndProcessed.py ${RELEASEDIR}/Configs/QingConfig ${MERGEDTRUTH_FILENAME}.pkl ${MINITREE_FILENAME} ${MERGED_FILENAME}.pkl;) &> ${MERGED_FILENAME}.log # Cleanup diff --git a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh index cf8439c..6ddfbf7 100644 --- a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh +++ b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh @@ -21,9 +21,6 @@ Nomial_g2=$3 RecoilType=ER -echo 'IfS1S2Correlation = ${IfS1S2Correlation}' -# enable s2 after pulse depending on the argument -# 1 for enable PMTAfterpulseEnableFlag=$4 S2AfterpulseEnableFlag=$5 @@ -81,6 +78,7 @@ PAX_FILENAME_WOPATH=${FILEROOT}_pax # pax processed data without path MERGEDTRUTH_FILENAME=${FILENAME}_merged_truth MERGED_FILENAME=${FILENAME}_merged HAX_FILENAME=${FILENAME}_hax # hax reduced data +MINITREE_FILENAME=${FILENAME}_pax_S1S2Properties.root CustomIniFilename=${RELEASEDIR}/NoS2Afterpulses.ini NoPMTAfterpulseIniFilename=${RELEASEDIR}/NoPMTAfterpulses.ini echo ${CustomIniFilename} @@ -119,20 +117,20 @@ python ${RELEASEDIR}/ConvertFaxTruthToPickle.py ${FAX_FILENAME} ${PKL_FILENAME} # pax stage (time paxer --ignore_rundb --input ${RAW_FILENAME} --config ${Detector} --output ${PAX_FILENAME};) &> ${PAX_FILENAME}.log -# hax stage -HAXPYTHON="import hax; " -HAXPYTHON+="hax.init(main_data_paths=['${OUTDIR}'], minitree_paths=['${OUTDIR}'], pax_version_policy = 'loose'); " -HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals']);" +#~ # hax stage +#~ HAXPYTHON="import hax; " +#~ HAXPYTHON+="hax.init(main_data_paths=['${OUTDIR}'], minitree_paths=['${OUTDIR}'], pax_version_policy = 'loose'); " +#~ HAXPYTHON+="hax.minitrees.load('${PAX_FILENAME##*/}', ['Basics', 'Fundamentals']);" -(time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log +#~ (time python -c "${HAXPYTHON}";) &> ${HAX_FILENAME}.log # custom minitree (time python ${RELEASEDIR}/ReduceDataNormal.py ${PAX_FILENAME_WOPATH} ${OUTDIR};) &> ${HAX_FILENAME}.log # merge -python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl -python ${RELEASEDIR}/MergeTruthAndProcessed.py ${MERGEDTRUTH_FILENAME}.pkl ${RELEASEDIR}/Configs/QingConfig ${MERGED_FILENAME}.pkl +(time python ${RELEASEDIR}/TruthSorting.py ${FAX_FILENAME}.root ${MERGEDTRUTH_FILENAME}.pkl;) &> ${MERGEDTRUTH_FILENAME}.log +(time python ${RELEASEDIR}/MergeTruthAndProcessed.py ${RELEASEDIR}/Configs/QingConfig ${MERGEDTRUTH_FILENAME}.pkl ${MINITREE_FILENAME} ${MERGED_FILENAME}.pkl;) &> ${MERGED_FILENAME}.log # Cleanup From 0581d0204b7ac1784de8f0ee194ceb6f9dd8fa98 Mon Sep 17 00:00:00 2001 From: Patrick de Perio Date: Tue, 28 Feb 2017 14:24:43 -0600 Subject: [PATCH 17/21] Update to pax v6.5.1 --- montecarlo/fax_waveform/run_fax.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index 36bfca3..5fed43b 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -31,7 +31,7 @@ S2AfterpulseEnableFlag=$6 # Select fax+pax version #~ PAXVERSION=head -PAXVERSION=v6.5.0 # temporary change +PAXVERSION=v6.5.1 # temporary change # Specify number of events NumEvents=$7 From 9b438627383ccfdef75fa1ad3c0d61fea3f05f13 Mon Sep 17 00:00:00 2001 From: Patrick de Perio Date: Tue, 28 Feb 2017 14:30:01 -0600 Subject: [PATCH 18/21] Update to pax v6.5.1 --- montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh index 6ddfbf7..8e92f27 100644 --- a/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh +++ b/montecarlo/fax_waveform/run_fax_AreaCorrelatedS1S2.sh @@ -26,7 +26,7 @@ S2AfterpulseEnableFlag=$5 # Select fax+pax version #~ PAXVERSION=head -PAXVERSION=v6.5.0 # temporary change +PAXVERSION=v6.5.1 # temporary change # Specify number of events NumEvents=$6 From 553a23cc6717b9b70b6dd34ce6191d02b4229364 Mon Sep 17 00:00:00 2001 From: Joey Howlett Date: Thu, 16 Mar 2017 08:40:48 -0500 Subject: [PATCH 19/21] removed unnecessary files --- montecarlo/fax_waveform/logs/170221_test.log | 153 ------------------ montecarlo/fax_waveform/logs/desc.txt | 1 + .../fax_waveform/processed_dataset_list.dat | 5 - 3 files changed, 1 insertion(+), 158 deletions(-) delete mode 100644 montecarlo/fax_waveform/logs/170221_test.log create mode 100644 montecarlo/fax_waveform/logs/desc.txt delete mode 100644 montecarlo/fax_waveform/processed_dataset_list.dat diff --git a/montecarlo/fax_waveform/logs/170221_test.log b/montecarlo/fax_waveform/logs/170221_test.log deleted file mode 100644 index c975e23..0000000 --- a/montecarlo/fax_waveform/logs/170221_test.log +++ /dev/null @@ -1,153 +0,0 @@ -1 -Submitted batch job 24206310 -/home/jh3226/testing_pax/montecarlo/fax_waveform -2 -Submitted batch job 24206311 -/home/jh3226/testing_pax/montecarlo/fax_waveform -3 -Submitted batch job 24206312 -/home/jh3226/testing_pax/montecarlo/fax_waveform -4 -Submitted batch job 24206313 -/home/jh3226/testing_pax/montecarlo/fax_waveform -5 -Submitted batch job 24206314 -/home/jh3226/testing_pax/montecarlo/fax_waveform -/home/jh3226/testing_pax/montecarlo/fax_waveform -Current job running number 1 -Current job running number 2 -Current job running number 3 -Current job running number 4 -Current job running number 5 -1 -Submitted batch job 24206319 -/home/jh3226/testing_pax/montecarlo/fax_waveform -2 -Submitted batch job 24206320 -/home/jh3226/testing_pax/montecarlo/fax_waveform -3 -Submitted batch job 24206322 -/home/jh3226/testing_pax/montecarlo/fax_waveform -4 -Submitted batch job 24206324 -/home/jh3226/testing_pax/montecarlo/fax_waveform -5 -Submitted batch job 24206325 -/home/jh3226/testing_pax/montecarlo/fax_waveform -/home/jh3226/testing_pax/montecarlo/fax_waveform -Current job running number 1 -Current job running number 2 -Current job running number 3 -Current job running number 4 -Current job running number 5 -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_truth.root to truth minitree dir -1 -Submitted batch job 24206376 -/home/jh3226/testing_pax/montecarlo/fax_waveform -2 -Submitted batch job 24206377 -/home/jh3226/testing_pax/montecarlo/fax_waveform -3 -Submitted batch job 24206378 -/home/jh3226/testing_pax/montecarlo/fax_waveform -4 -Submitted batch job 24206379 -/home/jh3226/testing_pax/montecarlo/fax_waveform -5 -Submitted batch job 24206380 -/home/jh3226/testing_pax/montecarlo/fax_waveform -To process -> FakeWaveform_XENON1T_000000_pax -Current job running number 1 -To process -> FakeWaveform_XENON1T_000003_pax -Current job running number 2 -To process -> FakeWaveform_XENON1T_000004_pax -Current job running number 3 -To process -> FakeWaveform_XENON1T_000001_pax -Current job running number 4 -To process -> FakeWaveform_XENON1T_000002_pax -Current job running number 5 -1 -Submitted batch job 24206599 -/home/jh3226/testing_pax/montecarlo/fax_waveform -2 -Submitted batch job 24206600 -/home/jh3226/testing_pax/montecarlo/fax_waveform -3 -Submitted batch job 24206601 -/home/jh3226/testing_pax/montecarlo/fax_waveform -4 -Submitted batch job 24206602 -/home/jh3226/testing_pax/montecarlo/fax_waveform -5 -Submitted batch job 24206604 -/home/jh3226/testing_pax/montecarlo/fax_waveform -/home/jh3226/testing_pax/montecarlo/fax_waveform -Current job running number 1 -Current job running number 2 -Current job running number 3 -Current job running number 4 -Current job running number 5 -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000000/FakeWaveform_XENON1T_000000_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000001/FakeWaveform_XENON1T_000001_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000002/FakeWaveform_XENON1T_000002_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000003/FakeWaveform_XENON1T_000003_truth.root to truth minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Basics.root to basics minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax_Fundamentals.root to processed minitree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_pax.root to pax tree dir -moving /project/lgrandi/jhowlett/170222_1108/000004/FakeWaveform_XENON1T_000004_truth.root to truth minitree dir -1 -Submitted batch job 24206622 -/home/jh3226/testing_pax/montecarlo/fax_waveform -2 -Submitted batch job 24206623 -/home/jh3226/testing_pax/montecarlo/fax_waveform -3 -Submitted batch job 24206624 -/home/jh3226/testing_pax/montecarlo/fax_waveform -4 -Submitted batch job 24206625 -/home/jh3226/testing_pax/montecarlo/fax_waveform -5 -Submitted batch job 24206626 -/home/jh3226/testing_pax/montecarlo/fax_waveform -To process -> FakeWaveform_XENON1T_000000_pax -Current job running number 1 -To process -> FakeWaveform_XENON1T_000003_pax -Current job running number 2 -To process -> FakeWaveform_XENON1T_000004_pax -Current job running number 3 -To process -> FakeWaveform_XENON1T_000001_pax -Current job running number 4 -To process -> FakeWaveform_XENON1T_000002_pax -Current job running number 5 diff --git a/montecarlo/fax_waveform/logs/desc.txt b/montecarlo/fax_waveform/logs/desc.txt new file mode 100644 index 0000000..535f0df --- /dev/null +++ b/montecarlo/fax_waveform/logs/desc.txt @@ -0,0 +1 @@ +placeholder, this is where log files will be diff --git a/montecarlo/fax_waveform/processed_dataset_list.dat b/montecarlo/fax_waveform/processed_dataset_list.dat deleted file mode 100644 index e1a5f6e..0000000 --- a/montecarlo/fax_waveform/processed_dataset_list.dat +++ /dev/null @@ -1,5 +0,0 @@ -FakeWaveform_XENON1T_000000_pax -FakeWaveform_XENON1T_000003_pax -FakeWaveform_XENON1T_000004_pax -FakeWaveform_XENON1T_000001_pax -FakeWaveform_XENON1T_000002_pax From 3b69749e1ef1fe138715f14ed83c42636c1238b3 Mon Sep 17 00:00:00 2001 From: Joey Howlett Date: Thu, 16 Mar 2017 08:48:11 -0500 Subject: [PATCH 20/21] fixed 2 conflicts with master --- montecarlo/fax_waveform/begin_production.py | 2 +- montecarlo/fax_waveform/run_fax.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/montecarlo/fax_waveform/begin_production.py b/montecarlo/fax_waveform/begin_production.py index 332fa2c..a913ee2 100644 --- a/montecarlo/fax_waveform/begin_production.py +++ b/montecarlo/fax_waveform/begin_production.py @@ -51,7 +51,7 @@ def fax_produce(process, head_dirname, username): production_commands.append('python BatchMergeTruthAndProcessed.py Configs/%s %s %s %s submission_merge/ %s 0 %s %s' % (configs[process['minitree_type']], truth_dirname, basics_dirname, merged_dirname, process['nodetype'], process['use_array_truth'], process['minitree_type'])) production_commands.append('python MergePickles.py %s' % (merged_dirname)) - for command in production_commands[-2:]: + for command in production_commands: wait_for_squeue(username, process['nodetype']) if process['minitree_type']!='0': if 'BatchReduce' in command: diff --git a/montecarlo/fax_waveform/run_fax.sh b/montecarlo/fax_waveform/run_fax.sh index 5fed43b..b3b9a61 100755 --- a/montecarlo/fax_waveform/run_fax.sh +++ b/montecarlo/fax_waveform/run_fax.sh @@ -43,8 +43,8 @@ SUBRUN=$9 # Setup the software CVMFSDIR=/cvmfs/xenon.opensciencegrid.org -#export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" -export PATH="/project/lgrandi/anaconda3/bin:$PATH" +export PATH="${CVMFSDIR}/releases/anaconda/2.4/bin:$PATH" +#export PATH="/project/lgrandi/anaconda3/bin:$PATH" source activate pax_${PAXVERSION} &> /dev/null # Use path of this script for Python scripts below From 921f0432944806145678ef401485901de277f560 Mon Sep 17 00:00:00 2001 From: Joey Howlett Date: Thu, 16 Mar 2017 08:49:33 -0500 Subject: [PATCH 21/21] updated to master version --- montecarlo/fax_waveform/TruthSorting_arrays.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/montecarlo/fax_waveform/TruthSorting_arrays.py b/montecarlo/fax_waveform/TruthSorting_arrays.py index 0725346..f2bfb99 100644 --- a/montecarlo/fax_waveform/TruthSorting_arrays.py +++ b/montecarlo/fax_waveform/TruthSorting_arrays.py @@ -61,7 +61,7 @@ # initialize Data for truth event_keys = ['index_truth', 'peaks_length'] -s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth', 'top_fraction'] +s1s2_keys = ['time_truth', 'time_std_truth', 'time_last_photon_truth', 'time_interaction_truth', 'area_truth', 'type_truth', 'x_truth', 'y_truth', 'z_truth', 'top_fraction', 'tag'] s2_only_keys = ['electron_time_truth', 'first_electron_time_truth', 'last_electron_time_truth'] for field in (event_keys + s1s2_keys + s2_only_keys): @@ -81,10 +81,17 @@ for field in (s1s2_keys + s2_only_keys): result[field] = [] + ifcounteds1 = 0 + while truth_tree.event==event_id: - tag = 0 # 0 for s1, 1 for s2 + tag = 0 # 0 for s1, 1 for s2, 2 for photoionization if not str(truth_tree.n_electrons)=='nan': tag = 1 + elif ifcounteds1==0: + tag=0 + ifcounteds1=1 + else: + tag=2 # fill these fields either way result['time_truth'].append(truth_tree.t_mean_photons) result['time_std_truth'].append(truth_tree.t_sigma_photons) @@ -97,8 +104,8 @@ result['z_truth'].append(truth_tree.z) result['top_fraction'].append(truth_tree.top_fraction) - if tag==0: - # peak is an S1 + if tag!=1: + # peak is not an s2 for s2_field in s2_only_keys: result[s2_field].append(float('nan')) else: @@ -106,6 +113,7 @@ result['electron_time_truth'].append(truth_tree.t_mean_electrons) result['first_electron_time_truth'].append(truth_tree.t_first_electron) result['last_electron_time_truth'].append(truth_tree.t_last_electron) + result['tag'].append(tag) iteration_id += 1 if iteration_id>=NumStepsInTruth: break