Fix: remove bad files & obsolete commands ci skip:syst

.github/workflows/BTA_workflow.yml

@@ -3,13 +3,24 @@ name: BTA output Workflows
 on:
   push:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/*BTA*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/BTA_workflow.yml'
   pull_request_target:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/*BTA*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/BTA_workflow.yml'
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     strategy:
       max-parallel: 4
       matrix:

.github/workflows/ctag_DY_workflow.yml

@@ -3,13 +3,24 @@ name: ctag DY+jets Workflow
 on:
   push:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag_*DY*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_DY_workflow.yml'
   pull_request_target:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag_*DY*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_DY_workflow.yml'
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     strategy:
       max-parallel: 4
       matrix:
@@ -87,9 +98,31 @@ jobs:
 
     - name: ctag muon DY workflows with correctionlib
       run: |
-        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3  --isSyst all --isArray  --year 2022 --overwrite
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ctag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3  --year 2022 $opts
 
 
     - name: ctag electron DY workflows with correctionlib
       run: |
-        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3  --isSyst all --isArray --year 2022 --overwrite
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ectag_DY_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3  --year 2022 $opts

.github/workflows/ctag_Wc_workflow.yml

@@ -3,13 +3,26 @@ name: ctag W+c Workflow
 on:
   push:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag_*Wc*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_Wc_workflow.yml'
+
   pull_request_target:
     branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag_*Wc*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_Wc_workflow.yml'
+
   workflow_dispatch:
 
 jobs:
   build:
     runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
     strategy:
       max-parallel: 4
       matrix:
@@ -52,7 +65,7 @@ jobs:
         conda info
         conda env list
         conda list
-        
+
     - name: Set up proxy
       # https://awesome-workshop.github.io/gitlab-cms/03-vomsproxy/index.html
       # continue-on-error: true
@@ -89,8 +102,30 @@ jobs:
 
     - name: ctag muon W+c workflows with correctionlib
       run: |
-        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative  --campaign Summer22Run3  --isSyst all --isArray  --overwrite
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ctag_Wc_sf --json metadata/test_bta_run3.json   --campaign Summer22Run3 --executor iterative  $opts
 
     - name: ctag electron W+c workflows with correctionlib
       run: |
-        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative  --campaign Summer22Run3  --isSyst all --isArray --overwrite
+        message=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ectag_Wc_sf --json metadata/test_bta_run3.json --executor iterative  --campaign Summer22Run3   --overwrite $opts

.github/workflows/ctag_ttbar_workflow.yml

@@ -0,0 +1,172 @@
+name: ctag ttbar workflows
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag*tt*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_ttbar_workflow.yml'
+  pull_request_target:
+    branches: [ master ]
+    paths:
+    - 'src/BTVNanoCommissioning/workflows/ctag*tt*'
+    - 'src/BTVNanoCommissioning/helpers/*'
+    - 'src/BTVNanoCommissioning/utils/*'
+    - '.github/workflows/ctag_ttbar_workflow.yml'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    if: ${{ !contains(github.event.head_commit.message, '[skip ci]') }}
+    strategy:
+      max-parallel: 4
+      matrix:
+        python-version: ["3.8","3.10"]
+
+    defaults:
+      run:
+        shell: "bash -l {0}"
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: update submodules
+      env:
+        SSHKEY: ${{ secrets.GIT_CERN_SSH_PRIVATE }}
+      run: |
+        mkdir  $HOME/.ssh
+        echo "$SSHKEY" > $HOME/.ssh/id_rsa
+        chmod 600  $HOME/.ssh/id_rsa
+        echo "HOST *" > ~/.ssh/config
+        echo "StrictHostKeyChecking no" >> ~/.ssh/config
+        git submodule update --init --recursive 
+    - uses: cvmfs-contrib/github-action-cvmfs@v2
+      with:
+        cvmfs_repositories: 'grid.cern.ch'
+
+    - name: Set conda environment
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        python-version: ${{ matrix.python-version }}
+        miniforge-variant: Mambaforge
+        channels: conda-forge,defaults
+        channel-priority: true
+        activate-environment: btv_coffea
+        environment-file: test_env.yml
+        auto-activate-base: false
+
+    - name: Verify environment
+      run: |
+        conda info
+        conda env list
+        conda list
+
+    - name: Set up proxy
+      # https://awesome-workshop.github.io/gitlab-cms/03-vomsproxy/index.html
+      # continue-on-error: true
+      env:
+        # To genereate secrets use (strip all \n)
+        # base64 -i ~/.globus/usercert.pem | awk NF=NF RS= OFS=
+        # base64 -i ~/.globus/userkey.pem | awk NF=NF RS= OFS=
+        # Cross check roundtrip by adding ``| base64 -d `` and see if same as input
+        GRID_USERKEY: ${{ secrets.GRID_USERKEY }}
+        GRID_USERCERT: ${{ secrets.GRID_USERCERT }}
+        # Read automatically by voms-proxy-init
+        X509_VOMS_DIR: /cvmfs/grid.cern.ch/etc/grid-security/vomsdir/
+        X509_VOMSES: /cvmfs/grid.cern.ch/etc/grid-security/vomses/
+        X509_DEFAULT_USER_CERT: $HOME/.globus/usercert.pem
+        X509_DEFAULT_USER_KEY: $HOME/.globus/userkey.pem
+      run: |
+        mkdir $HOME/.globus
+        printf $GRID_USERKEY | base64 -d > $HOME/.globus/userkey.pem
+        printf $GRID_USERCERT | base64 -d > $HOME/.globus/usercert.pem
+        # DEBUG: dump decoded cert, cert is public, but don't dump key!
+        # base64 -i $HOME/.globus/usercert.pem
+        chmod 400 $HOME/.globus/userkey.pem
+        openssl rand -out $HOME/.rnd  -hex 256
+        printf "${{secrets.GRID_PASSWORD}}" | voms-proxy-init --voms cms --vomses ${X509_VOMSES} --debug --pwstdin
+
+    - name: Test xrootd
+      run: |
+        xrdcp root://eoscms.cern.ch//eos/cms/store/group/phys_btag/nano-commissioning/test_w_dj.root .
+
+    - name: Install Repo
+      run: |
+        pip install -e .
+
+
+    - name: ctag semileptonic muon ttbar workflows with correctionlib
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ctag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3 $opts
+
+    - name: ctag semileptonic electron ttbar workflows with correctionlib
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi 
+        python runner.py --workflow ectag_ttsemilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3 $opts
+
+    - name: ctag dileptonic muon ttbar workflows with correctionlib
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi 
+        python runner.py --workflow ctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3 $opts
+    - name: ctag dileptonic electron ttbar workflows with correctionlib 
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow ectag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3 $opts
+    - name: ctag dileptonic emu ttbar workflows with correctionlib
+      run: |
+        string=$(git log -1 --pretty=format:'%s')
+        if [[ $string == *"ci:skip array"* ]]; then
+        opts=$(echo "$opts" | sed 's/--isArray //g')
+        fi
+        if [[ $string == *"ci:skip syst"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all//g')
+        elif [[ $string == *"ci:JERC_split"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst JERC_split/g')
+        elif [[ $string == *"ci:weight_only"* ]]; then
+            opts=$(echo "$opts" | sed 's/--isSyst all/--isSyst weight_only/g') 
+        fi
+        python runner.py --workflow emctag_ttdilep_sf --json metadata/test_bta_run3.json --limit 1 --executor iterative --campaign Summer22Run3 $opts
+