Beamline neural network reconstruction training

.gitignore

@@ -36,6 +36,13 @@ __pycache__/
 *$py.class
 .ipynb_checkpoints
 
+# results and simulation output
+results/
+sim_output/
+.snakemake/
+calibrations/
+fieldmaps/
+
 # test for calorimeter
 calorimeters/test/
 *.d

.gitlab-ci.yml

@@ -122,70 +122,72 @@ common:setup:
       - runner_system_failure
 
 include: 
-  - local: 'benchmarks/backgrounds/config.yml'
-  - local: 'benchmarks/backwards_ecal/config.yml'
+  # - local: 'benchmarks/backgrounds/config.yml'
+  # - local: 'benchmarks/backwards_ecal/config.yml'
   - local: 'benchmarks/beamline/config.yml'
-  - local: 'benchmarks/calo_pid/config.yml'
-  - local: 'benchmarks/campaign/config.yml'
-  - local: 'benchmarks/ecal_gaps/config.yml'
-  - local: 'benchmarks/tracking_detectors/config.yml'
-  - local: 'benchmarks/tracking_performances/config.yml'
-  - local: 'benchmarks/tracking_performances_dis/config.yml'
-  - local: 'benchmarks/barrel_ecal/config.yml'
-  - local: 'benchmarks/lfhcal/config.yml'
-  - local: 'benchmarks/zdc/config.yml'
-  - local: 'benchmarks/zdc_lyso/config.yml'
-  - local: 'benchmarks/zdc_neutron/config.yml'
-  - local: 'benchmarks/zdc_photon/config.yml'
-  - local: 'benchmarks/zdc_pi0/config.yml'
-  - local: 'benchmarks/material_scan/config.yml'
-  - local: 'benchmarks/pid/config.yml'
-  - local: 'benchmarks/rich/config.yml'
-  - local: 'benchmarks/b0_tracker/config.yml'
-  - local: 'benchmarks/insert_muon/config.yml'
-  - local: 'benchmarks/insert_tau/config.yml'
-  - local: 'benchmarks/zdc_sigma/config.yml'
-  - local: 'benchmarks/zdc_lambda/config.yml'
-  - local: 'benchmarks/insert_neutron/config.yml'
-  - local: 'benchmarks/femc_electron/config.yml'
-  - local: 'benchmarks/femc_photon/config.yml'
-  - local: 'benchmarks/femc_pi0/config.yml'
-  - local: 'benchmarks/nhcal_acceptance/config.yml'
-  - local: 'benchmarks/nhcal_basic_distribution/config.yml'
+  # - local: 'benchmarks/calo_pid/config.yml'
+  # - local: 'benchmarks/campaign/config.yml'
+  # - local: 'benchmarks/ecal_gaps/config.yml'
+  - local: 'benchmarks/lowq2_reconstruction/config.yml'
+  # - local: 'benchmarks/tracking_detectors/config.yml'
+  # - local: 'benchmarks/tracking_performances/config.yml'
+  # - local: 'benchmarks/tracking_performances_dis/config.yml'
+  # - local: 'benchmarks/barrel_ecal/config.yml'
+  # - local: 'benchmarks/lfhcal/config.yml'
+  # - local: 'benchmarks/zdc/config.yml'
+  # - local: 'benchmarks/zdc_lyso/config.yml'
+  # - local: 'benchmarks/zdc_neutron/config.yml'
+  # - local: 'benchmarks/zdc_photon/config.yml'
+  # - local: 'benchmarks/zdc_pi0/config.yml'
+  # - local: 'benchmarks/material_scan/config.yml'
+  # - local: 'benchmarks/pid/config.yml'
+  # - local: 'benchmarks/rich/config.yml'
+  # - local: 'benchmarks/b0_tracker/config.yml'
+  # - local: 'benchmarks/insert_muon/config.yml'
+  # - local: 'benchmarks/insert_tau/config.yml'
+  # - local: 'benchmarks/zdc_sigma/config.yml'
+  # - local: 'benchmarks/zdc_lambda/config.yml'
+  # - local: 'benchmarks/insert_neutron/config.yml'
+  # - local: 'benchmarks/femc_electron/config.yml'
+  # - local: 'benchmarks/femc_photon/config.yml'
+  # - local: 'benchmarks/femc_pi0/config.yml'
+  # - local: 'benchmarks/nhcal_acceptance/config.yml'
+  # - local: 'benchmarks/nhcal_basic_distribution/config.yml'
 
 deploy_results:
   allow_failure: true
   stage: deploy
   needs:
-    - "collect_results:backgrounds"
-    - "collect_results:backwards_ecal"
-    - "collect_results:barrel_ecal"
-    - "collect_results:beamline"
-    - "collect_results:calo_pid"
-    - "collect_results:campaign"
-    - "collect_results:ecal_gaps"
-    - "collect_results:lfhcal"
-    - "collect_results:material_scan"
-    - "collect_results:pid"
-    - "collect_results:rich"
-    - "collect_results:tracking_performance"
-    - "collect_results:tracking_performance_campaigns"
-    - "collect_results:zdc_sigma"
-    - "collect_results:zdc_lambda"
-    - "collect_results:insert_neutron"
-    - "collect_results:tracking_performances_dis"
-    - "collect_results:zdc"
-    - "collect_results:zdc_lyso"
-    - "collect_results:zdc_neutron"
-    - "collect_results:insert_muon"
-    - "collect_results:insert_tau"
-    - "collect_results:zdc_photon"
-    - "collect_results:zdc_pi0"
-    - "collect_results:femc_electron"
-    - "collect_results:femc_photon"
-    - "collect_results:femc_pi0"
-    - "collect_results:nhcal_acceptance"
-    - "collect_results:nhcal_basic_distribution"
+    # - "collect_results:backgrounds"
+    # - "collect_results:backwards_ecal"
+    # - "collect_results:barrel_ecal"
+    # - "collect_results:beamline"
+    # - "collect_results:calo_pid"
+    # - "collect_results:campaign"
+    # - "collect_results:ecal_gaps"
+    # - "collect_results:lfhcal"
+    - "collect_results:lowq2_reconstruction"
+    # - "collect_results:material_scan"
+    # - "collect_results:pid"
+    # - "collect_results:rich"
+    # - "collect_results:tracking_performance"
+    # - "collect_results:tracking_performance_campaigns"
+    # - "collect_results:zdc_sigma"
+    # - "collect_results:zdc_lambda"
+    # - "collect_results:insert_neutron"
+    # - "collect_results:tracking_performances_dis"
+    # - "collect_results:zdc"
+    # - "collect_results:zdc_lyso"
+    # - "collect_results:zdc_neutron"
+    # - "collect_results:insert_muon"
+    # - "collect_results:insert_tau"
+    # - "collect_results:zdc_photon"
+    # - "collect_results:zdc_pi0"
+    # - "collect_results:femc_electron"
+    # - "collect_results:femc_photon"
+    # - "collect_results:femc_pi0"
+    # - "collect_results:nhcal_acceptance"
+    # - "collect_results:nhcal_basic_distribution"
   script:
     - snakemake $SNAKEMAKE_FLAGS --cores 1 results/metadata.json
     - find results -print | sort | tee summary.txt

Snakefile

@@ -30,31 +30,32 @@ def find_epic_libraries():
     return libs
 
 
-include: "benchmarks/backgrounds/Snakefile"
-include: "benchmarks/backwards_ecal/Snakefile"
-include: "benchmarks/barrel_ecal/Snakefile"
+# include: "benchmarks/backgrounds/Snakefile"
+# include: "benchmarks/backwards_ecal/Snakefile"
+# include: "benchmarks/barrel_ecal/Snakefile"
 include: "benchmarks/beamline/Snakefile"
-include: "benchmarks/calo_pid/Snakefile"
-include: "benchmarks/campaign/Snakefile"
-include: "benchmarks/ecal_gaps/Snakefile"
-include: "benchmarks/material_scan/Snakefile"
-include: "benchmarks/tracking_performances/Snakefile"
-include: "benchmarks/tracking_performances_dis/Snakefile"
-include: "benchmarks/lfhcal/Snakefile"
-include: "benchmarks/zdc_lyso/Snakefile"
-include: "benchmarks/zdc_neutron/Snakefile"
-include: "benchmarks/insert_muon/Snakefile"
-include: "benchmarks/zdc_lambda/Snakefile"
-include: "benchmarks/zdc_photon/Snakefile"
-include: "benchmarks/zdc_pi0/Snakefile"
-include: "benchmarks/zdc_sigma/Snakefile"
-include: "benchmarks/insert_neutron/Snakefile"
-include: "benchmarks/insert_tau/Snakefile"
-include: "benchmarks/femc_electron/Snakefile"
-include: "benchmarks/femc_photon/Snakefile"
-include: "benchmarks/femc_pi0/Snakefile"
-include: "benchmarks/nhcal_acceptance/Snakefile"
-include: "benchmarks/nhcal_basic_distribution/Snakefile"
+# include: "benchmarks/calo_pid/Snakefile"
+# include: "benchmarks/campaign/Snakefile"
+# include: "benchmarks/ecal_gaps/Snakefile"
+include : "benchmarks/lowq2_reconstruction/Snakefile"
+# include: "benchmarks/material_scan/Snakefile"
+# include: "benchmarks/tracking_performances/Snakefile"
+# include: "benchmarks/tracking_performances_dis/Snakefile"
+# include: "benchmarks/lfhcal/Snakefile"
+# include: "benchmarks/zdc_lyso/Snakefile"
+# include: "benchmarks/zdc_neutron/Snakefile"
+# include: "benchmarks/insert_muon/Snakefile"
+# include: "benchmarks/zdc_lambda/Snakefile"
+# include: "benchmarks/zdc_photon/Snakefile"
+# include: "benchmarks/zdc_pi0/Snakefile"
+# include: "benchmarks/zdc_sigma/Snakefile"
+# include: "benchmarks/insert_neutron/Snakefile"
+# include: "benchmarks/insert_tau/Snakefile"
+# include: "benchmarks/femc_electron/Snakefile"
+# include: "benchmarks/femc_photon/Snakefile"
+# include: "benchmarks/femc_pi0/Snakefile"
+# include: "benchmarks/nhcal_acceptance/Snakefile"
+# include: "benchmarks/nhcal_basic_distribution/Snakefile"
 
 use_s3 = config["remote_provider"].lower() == "s3"
 use_xrootd = config["remote_provider"].lower() == "xrootd"

benchmarks/beamline/Snakefile

@@ -1,6 +1,10 @@
 SIMOUTDIR="sim_output/beamline/"
 ANALYSISDIR=SIMOUTDIR+"analysis/"
 
+##########################################################################################
+### Rules for checking the steering of the electron beam through the magnets
+##########################################################################################
+
 rule beamline_steering_sim:
     input:
         warmup="warmup/epic_ip6_extended.edm4hep.root",
@@ -20,25 +24,6 @@ rule beamline_steering_sim:
             --physics.rangecut 100*m
         """
 
-rule beamline_acceptance_sim:
-    input:
-        warmup="warmup/epic_ip6_extended.edm4hep.root",
-        macro=workflow.source_path("acceptanceGPS.mac"),
-    output:
-        SIMOUTDIR+"acceptanceTest{CAMPAIGN}.edm4hep.root",
-    shell:
-        """
-            exec npsim \
-            --runType run \
-            --random.seed 1 \
-            --enableG4GPS \
-            --macroFile {input.macro} \
-            --compactFile $DETECTOR_PATH/epic_ip6_extended.xml \
-            --printLevel WARNING \
-            --outputFile {output} \
-            --physics.rangecut 100*m
-        """
-
 rule beamline_steering_analysis:
     input:
         warmup="warmup/epic_ip6_extended.edm4hep.root",
@@ -63,6 +48,28 @@ rule beamline_steering_analysis:
             "{output.pipe_parameter_canvas}")'
         """
 
+##########################################################################################
+### Rules for checking the acceptance of electrons at each stage of the beamline
+##########################################################################################
+
+rule beamline_acceptance_sim:
+    input:
+        warmup="warmup/epic_ip6_extended.edm4hep.root",
+        macro=workflow.source_path("acceptanceGPS.mac"),
+    output:
+        SIMOUTDIR+"acceptanceTest{CAMPAIGN}.edm4hep.root",
+    shell:
+        """
+            exec npsim \
+            --runType run \
+            --random.seed 1 \
+            --enableG4GPS \
+            --macroFile {input.macro} \
+            --compactFile $DETECTOR_PATH/epic_ip6_extended.xml \
+            --outputFile {output} \
+            --physics.rangecut 100*m
+        """
+
 rule beamline_acceptance_analysis:
     input:
         warmup="warmup/epic_ip6_extended.edm4hep.root",
@@ -80,9 +87,12 @@ rule beamline_acceptance_analysis:
     shell:
         """
             root -l -b -q '{input.script}("{input.data}", "{output.rootfile}", "{params.xml}", "{output.beampipe_canvas}","{output.etheta_canvas}","{output.etheta_acceptance_canvas}",
-            "{output.entries_canvas}")'
+            "{output.entrys_canvas}")'
         """
 
+##########################################################################################
+# Combine results
+##########################################################################################
 rule beamline:
     input:
         ANALYSISDIR+"beamlineTestAnalysis{CAMPAIGN}.root",
@@ -96,15 +106,18 @@ rule beamline:
         ANALYSISDIR+"acceptance_in_beampipe_{CAMPAIGN}.png",
         ANALYSISDIR+"acceptance_energy_theta_{CAMPAIGN}.png",
         ANALYSISDIR+"acceptance_energy_theta_acceptance_{CAMPAIGN}.png",     
-        ANALYSISDIR+"acceptance_entries_{CAMPAIGN}.png", 
+        ANALYSISDIR+"acceptance_entries_{CAMPAIGN}.png"
     output:
-        directory("results/beamline/{CAMPAIGN}/")
+        directory("results/beamline/steering_{CAMPAIGN}/")
     shell:
         """
             mkdir {output}        
-            cp {input} {output}
+            cp -r {input} {output}
         """
 
+##########################################################################################
+# Defualt running
+##########################################################################################
 rule beamline_local:
     input:
         "results/beamline/local/"

benchmarks/beamline/beamlineAnalysis.C

@@ -316,6 +316,8 @@ int beamlineAnalysis(   TString inFile          = "/scratch/EIC/G4out/beamline/b
             std::cout << "Warning: Only " << h->GetEntries()/nEntries << " of particles contributing to histogram " <<  name
                       << " , which is below the accepted threshold of " << acceptableEntries/nEntries << std::endl;
             pass = 1;
+        } else{
+            std::cout << "Histogram " << name << " has " << h->GetEntries() << " entries." << std::endl;
         }
 
         // Get the pipe radius for this histogram

benchmarks/lowq2_reconstruction/LoadData.py

@@ -0,0 +1,15 @@
+import uproot
+import awkward as ak
+
+def create_arrays(dataFiles,featureName="_TaggerTrackerFeatureTensor_floatData",targetName="_TaggerTrackerTargetTensor_floatData", entries=None, treeName="events"):
+
+    # List of branches to load
+    branches = [featureName,targetName]
+
+    # Load data from concatenated list of files
+    data = uproot.concatenate([f"{file}:{treeName}" for file in dataFiles], branches, entry_stop=entries, library="ak")
+
+    input_data  = data[featureName]
+    target_data = data[targetName]
+
+    return input_data, target_data