Merge branch 'btv_master' of https://github.com/Ming-Yan/BTVNanoCommi…

…ssioning into coffea_update

.github/workflows/BTA_workflow.yml

@@ -36,7 +36,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-
     - name: Merge PR locally
       if: github.event_name == 'pull_request_target'
       id: merge
@@ -105,12 +104,12 @@ jobs:
         pip install -e .
     - name: BTA addAllTracks workflow test(JP)
       run: |
-         python runner.py --wf BTA_addAllTracks --json metadata/test_bta_run3.json --executor iterative --overwrite 
+         python runner.py --wf BTA_addAllTracks --json metadata/test_bta_run3.json --executor iterative --overwrite --campaign Summer23 --year 2023
     - name: BTA addPFMuons workflow test(BTA for SF)
       run: |
-         python runner.py --wf BTA_addPFMuons --json metadata/test_bta_run3.json --executor iterative --overwrite 
+         python runner.py --wf BTA_addPFMuons --json metadata/test_bta_run3.json --executor iterative --overwrite --campaign Summer23 --year 2023
     
     - name: BTA_ttbar workflow test
       run: |
-         python runner.py --wf BTA_ttbar --json metadata/test_bta_run3.json  --executor iterative --overwrite
+         python runner.py --wf BTA_ttbar --json metadata/test_bta_run3.json  --executor iterative --overwrite --campaign Summer23 --year 2023
 

.github/workflows/QCD_workflow.yml

@@ -105,7 +105,7 @@ jobs:
         pip install -e .
     
 
-    - name: QCD workflows with correctionlib
+    - name: QCD workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -118,7 +118,7 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow QCD_sf --json metadata/test_bta_run3.json  --executor iterative  $opts
+        python runner.py --workflow QCD_sf --json metadata/test_bta_run3.json  --executor iterative  --campaign Summer23 --year 2023 $opts
     
     - name: QCD soft mu workflows with correctionlib
       run: |
@@ -134,5 +134,5 @@ jobs:
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
 
-        python runner.py --workflow QCD_smu_sf --json metadata/test_bta_run3.json --executor iterative  --overwrite $opts
+        python runner.py --workflow QCD_smu_sf --json metadata/test_bta_run3.json --executor iterative  --campaign Summer23 --year 2023 --overwrite $opts
 

.github/workflows/ctag_DY_workflow.yml

@@ -36,7 +36,6 @@ jobs:
 
     steps:
     - uses: actions/checkout@v2
-
     - name: Merge PR locally
       if: github.event_name == 'pull_request_target'
       id: merge
@@ -103,7 +102,7 @@ jobs:
         pip install -e .
     
 
-    - name: ctag muon DY workflows with correctionlib
+    - name: ctag muon DY workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -116,10 +115,10 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative  $opts
+        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023  $opts
     
 
-    - name: ctag electron DY workflows with correctionlib
+    - name: ctag electron DY workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -132,4 +131,4 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts 

.github/workflows/ctag_Wc_workflow.yml

@@ -105,7 +105,7 @@ jobs:
         pip install -e .
     
 
-    - name: ctag muon W+c workflows with correctionlib
+    - name: ctag muon W+c workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -118,9 +118,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json  --executor iterative  $opts
+        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json  --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag electron W+c workflows with correctionlib
+    - name: ctag electron W+c workflows
       run: |
         message=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -133,4 +133,4 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --executor iterative  --overwrite $opts
+        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --executor iterative  --overwrite --campaign Summer23 --year 2023 $opts

.github/workflows/ctag_ttbar_workflow.yml

@@ -104,7 +104,7 @@ jobs:
         pip install -e .
 
     
-    - name: ctag semileptonic muon ttbar workflows with correctionlib
+    - name: ctag semileptonic muon ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -117,9 +117,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ctag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ctag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag semileptonic electron ttbar workflows with correctionlib
+    - name: ctag semileptonic electron ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -132,9 +132,9 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi 
-        python runner.py --workflow ectag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ectag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
     
-    - name: ctag dileptonic muon ttbar workflows with correctionlib
+    - name: ctag dileptonic muon ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -147,8 +147,8 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi 
-        python runner.py --workflow ctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
-    - name: ctag dileptonic electron ttbar workflows with correctionlib 
+        python runner.py --workflow ctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
+    - name: ctag dileptonic electron ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -161,8 +161,8 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ectag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
-    - name: ctag dileptonic emu ttbar workflows with correctionlib
+        python runner.py --workflow ectag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
+    - name: ctag dileptonic emu ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -175,5 +175,5 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow emctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow emctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
 

.github/workflows/ttbar_workflow.yml

@@ -102,7 +102,7 @@ jobs:
       run: |
         pip install -e .
 
-    - name:  btag semileptonic ttbar workflows with correctionlib
+    - name:  btag semileptonic ttbar workflows
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -115,7 +115,7 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023  $opts
 
     - name:  semileptonic + c ttbar workflows with correctionlib
       run: |
@@ -130,10 +130,10 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow c_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow c_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023  $opts
     
 
-    - name: btag dileptonic ttbar workflows with correctionlib
+    - name: btag dileptonic ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         if [[ $string == *"ci:skip array"* ]]; then
@@ -146,5 +146,5 @@ jobs:
         elif [[ $string == *"ci:weight_only"* ]]; then
             opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
         fi
-        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer23 --year 2023 $opts
 

.github/workflows/validation.yml

@@ -103,9 +103,9 @@ jobs:
         pip install -e .
 
    
-    - name: btag dileptonic ttbar workflows with correctionlib
+    - name: btag dileptonic ttbar workflows 
       run: |
         string=$(git log -1 --pretty=format:'%s')
         
-        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative $opts
+        python runner.py --workflow ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --year 2023 --campaign Summer23 $opts
 

.gitlab-ci.yml

@@ -0,0 +1,29 @@
+
+# This file is a template, and might need editing before it works on your project.
+# This is a sample GitLab CI/CD configuration file that should run without any modifications.
+# It demonstrates a basic 3 stage CI/CD pipeline. Instead of real tests or scripts,
+# it uses echo commands to simulate the pipeline execution.
+#
+# A pipeline is composed of independent jobs that run scripts, grouped into stages.
+# Stages run in sequential order, but jobs within stages run in parallel.
+#
+# For more information, see: https://docs.gitlab.com/ee/ci/yaml/index.html#stages
+#
+# You can copy and paste this template into a new `.gitlab-ci.yml` file.
+# You should not add this template to an existing `.gitlab-ci.yml` file by using the `include:` keyword.
+#
+# To contribute improvements to CI/CD templates, please follow the Development guide at:
+# https://docs.gitlab.com/ee/development/cicd/templates.html
+# This specific template is located at:
+# https://gitlab.com/gitlab-org/gitlab/-/blob/master/lib/gitlab/ci/templates/Getting-Started.gitlab-ci.yml
+
+stages:          # List of stages for jobs, and their order of execution
+  - deploy
+
+deploy-job:      # This job runs in the deploy stage.
+  stage: deploy  # It only runs when *both* jobs in the test stage complete successfully.
+  script:
+    - 'curl --fail --request POST --form token=$MY_TRIGGER_TOKEN --form ref=master "https://gitlab.cern.ch/cms-analysis/btv/software-and-algorithms/autobtv/trigger/pipeline"'
+  rules:
+    - if: $CI_COMMIT_TAG
+  environment: production

README.md

@@ -16,13 +16,18 @@ Detailed documentation in [btv-wiki](https://btv-wiki.docs.cern.ch/SoftwareAlgor
 ### Setup 
 
 :heavy_exclamation_mark: suggested to install under `bash` environment
-:heavy_exclamation_mark: :heavy_exclamation_mark: not fully supported in EL9 machines yet, recommended to run in EL7 or EL8
+
 ```
 # only first time, including submodules
 git clone --recursive [email protected]:cms-btv-pog/BTVNanoCommissioning.git 
 
 # activate enviroment once you have coffea framework 
 conda activate btv_coffea
+# Or if you are using lxplus, can use exist framework
+conda activate /eos/user/m/milee/miniconda3/envs/btv_coffea
+
+# when modifying the framework
+pip install -e .
 ```
 ### Coffea installation with Micromamba
 For installing Micromamba, see [[here](https://mamba.readthedocs.io/en/latest/installation/micromamba-installation.html)]

condor/execute.sh

@@ -15,6 +15,8 @@ WORKDIR=`pwd`
 # Set up mamba environment
 ## Interactive bash script with fallback pointing to $HOME, hence setting $PWD of worker node as $HOME
 export HOME=`pwd`
+
+if ${ARGS[remoteRepo]}!="":
 wget -L micro.mamba.pm/install.sh
 chmod +x install.sh
 ## FIXME parsing arguments does not work. will use defaults in install.sh instead, see https://github.com/mamba-org/micromamba-releases/blob/main/install.sh 
@@ -32,7 +34,7 @@ fi
 
 # Get arguments
 declare -A ARGS
-for key in workflow output samplejson year campaign isSyst isArray noHist overwrite voms chunk skipbadfiles outputXrootdDir remoteRepo; do
+for key in workflow output samplejson year campaign isSyst isArray noHist overwrite voms chunk skipbadfiles outputDir remoteRepo; do
     ARGS[$key]=$(jq -r ".$key" $WORKDIR/arguments.json)
 done
 
@@ -86,25 +88,25 @@ echo "Now launching: python runner.py $OPTS"
 python runner.py $OPTS
 
 # Transfer output
-if [[ ${ARGS[outputXrootdDir]} == root://* ]]; then
+if [[ ${ARGS[outputDir]} == root://* ]]; then
 
-    xrdcp --silent -p -f -r hists_* ${ARGS[outputXrootdDir]}/
+    xrdcp --silent -p -f -r hists_* ${ARGS[outputDir]}/
     if [[ "$OPTS" == *"isArray"* ]]; then
-	xrdcp --silent -p -f -r arrays_* ${ARGS[outputXrootdDir]}/
+	xrdcp --silent -p -f -r arrays_* ${ARGS[outputDir]}/
     fi
 else
-    mkdir -p ${ARGS[outputXrootdDir]}
-    cp -p -f -r hists_* ${ARGS[outputXrootdDir]}/
+    mkdir -p ${ARGS[outputDir]}
+    cp -p -f -r hists_* ${ARGS[outputDir]}/
     if [[ "$OPTS" == *"isArray"* ]]; then
-	cp -p -f -r arrays_* ${ARGS[outputXrootdDir]}/
+	cp -p -f -r arrays_* ${ARGS[outputDir]}/
     fi
 fi
 
 ### one can also consider origanizing the root files in the subdirectory structure ###
 # for filename in `\ls *.root`; do
 #     SAMPLENAME=$(echo "$filename" | sed -E 's/(.*)_f[0-9-]+_[0-9]+\.root/\1/')
 #     # SAMPLENAME=$(echo "$filename" | sed -E 's/(.*)_[0-9a-z]{9}-[0-9a-z]{4}-.*\.root/\1/')
-#     xrdcp --silent -p -f $filename ${ARGS[outputXrootdDir]}/$SAMPLENAME/
+#     xrdcp --silent -p -f $filename ${ARGS[outputDir]}/$SAMPLENAME/
 # done
 
 touch $WORKDIR/.success

condor/submitter.py

@@ -29,8 +29,8 @@ def get_condor_submitter_parser(parser):
         help="Number of files proceed per condor job",
     )
     parser.add_argument(
-        "--outputXrootdDir",
-        help="Output directory for xrootd files. Start with root://",
+        "--outputDir",
+        help="Output directory",
         required=True,
     )
     parser.add_argument(
@@ -216,7 +216,6 @@ def get_main_parser():
     jdl_template = """Universe   = vanilla
 Executable = {executable}
 
-+ProjectName="cms.org.cern"
 
 Arguments = $(JOBNUM)
 
@@ -245,7 +244,7 @@ def get_main_parser():
     )
     with open(os.path.join(job_dir, "submit.jdl"), "w") as f:
         f.write(jdl_template)
-
+    os.system(f"condor_submit {job_dir}/submit.jdl")
     print(
         f"Setup completed. Now submit the condor jobs by:\n  condor_submit {job_dir}/submit.jdl"
     )