config.yaml

# ======================================================================
# Download resources
#   - ilastik classifiers
#   - example data: if using data supplied by user, set `download` to 
#     false
# ======================================================================
resources:
  ilastik:
    u2os_nucleoli_bfp_v1: 'https://www.dropbox.com/s/ryoma1qvklinxn6/u2os_nucleoli_bfp_v1.ilp?dl=1'
    u2os_nucleoli_bfp_v2: 'https://www.dropbox.com/scl/fi/vv4n4246dsxa472wvpbeg/u2os_nucleoli_bfp_v2.ilp?rlkey=8co2dui00tegjxtfef86gnl5j&dl=1'
  example_data:
    name: alu-repo-sample-data.zip
    url: 'https://www.dropbox.com/scl/fo/mwzas1osvhsdv49jhf4gm/h?rlkey=383hoze85fb5pueq0dgdkv7we&dl=1'
    download: true


# ======================================================================
# Experiment-specific configurations
# ======================================================================

# No need to quote the strings in yaml files, but do so when special 
# characters are used. More importantly, even if they are quoted here,
# they still need to be quoted again when used in the "shell" derivative.
input:
  samplesheet_path: resources/example_data/samplesheet.csv
  image_path:
    dir: resources/example_data
    glob: '*002_Deno.nd2'
    regex: 'Well(?P<WellID>[A-H]\d{2})_\w+_fov(?P<FovID>\d{3})_snap-nodelay-snap002_Deno.nd2'
  datefmt: # leave blank if not used
  # channels has to be in the order as how they present in the raw file
  channels: 
    - oc: "640"
      fluoro: miRFP670
      colormap: magenta
      background: 100 
    - oc: "488"
      fluoro: sfGFP
      colormap: green
      background: 100
  bitdepth: 16

parse_metadata:
  nafilter: strict
  verbose: True
  patch: True

# 1. channels has to be in the order as how they have been specified 
#    in 'input' module above.
# 2. if you don't specify all the channels from the movies, those 
#    unspecified channels will be lost, creating unexpected behaviors
#    downstream.
register_nucleus:
  transformation: RIGID_BODY
  mode: previous
  registers:
    - target: miRFP670
      by: sfGFP
    - target: sfGFP
      by: miRFP670

segment_nuclei_in_time:
  channel: sfGFP
  diameter: 300
  downsample: 10

segment_hc:
  channel: miRFP670
  hc:
    blur_sigma: 1.0
    zscore_threshold: 2.0
    clean_disk_size: 2
    clean_min_size: 50
    time_threshold: 0.5

normalize:
  structure:
    - nucleus
  method: zscore
  # nested as dict. leave it empty if not needed
  method_kwargs:

sn:
  predict_nucleoli:
    channel_index: 1
    model_name: u2os_nucleoli_bfp_v2

measure:
  structure:
    - nucleus
    # - nucleoplasm

piv:
  pkg_path: "external/MatPIV-1.7"
  channel: 
    - miRFP670
    - sfGFP
  movie_meta:
    PIXELSIZE: 0.065
    DT: 0.5
    LAGS_EXPRESSION: "unique(ceil(logspace(0,log10(50),15)))" 
  protocol: matpiv_v2
  protocol_configs:
    matpiv_v2:
      CONFIG:
        PREPROCESS:
          BACK: 100
        PIV:
          WINSIZE: 16
          OVERLAP: 0.75
          MODE: 'single'
        FILTERS:
          SNR: 1.1
          PKH: 0.3
          GLOBAL: 3

# ----------------------------------------------------------------
# MSND analysis
#
# For the "protocols" block, each item corresponds to a specific
# MSND analysis protocol, including input data names and filename
# patterns, preprocessing steps, and MSND type (0-, 1-, 2-d).
#
# Make sure to also update configs for msnd_post rule below for
# each of the protocol specified in this block.
# ----------------------------------------------------------------
msnd:
  # Source movie's channel which PIV generated from
  channel:
    - miRFP670
    - sfGFP

  # Each protocol contains "data", "preprocess", and "process" blocks
  # - For data, each item is a dict
  #   <name>:                  # becomes key in data (i.e., data[<name>])
  #      path: <path pattern>  # including wildcards pattern
  #                            # if left blank, will try to use default
  #      type: <type>          # determines how to read data in
  # - For preprocess, each item is a dict
  #   name: <name>             # a descriptive name, not used in code
  #   input: <input_name>      # refers to data[<input_name>]. possible
  #                            # options depend on those defined in "data"
  #   output: <output_name>    # can be referred to in a later preprocess
  #                            # for image and piv, make sure to name the
  #                            # last one *_final
  #   steps:                   # list of dict
  #     - func: <data_type>_<func_name> # see msnd_utils.py for options
  #       params:              # dict of dict
  #                            # for params to be passed as kwargs
  # - For process,
  #   name: <name>             # possible options now: 0d, 1d, 2d
  #   params:                  # dict of dict
  #                            # for params to be passed as kwargs
  protocols:
    # regular over all nucleoplasmic pixels
    normal_0d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
      process:
        name: 0d
        params:

    # 1D intensity-binned
    eachlevel_1d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
      process:
        name: 1d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25

    # 2D intensity-binned
    eachlevel_2d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: ['u', 'v']
                max: 4
      process:
        name: 2d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25

    # nucleoli and heterochromatin excluded
    euchromatin_0d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        mask_hc:
          path: "results/segmentation/hc/{RoiUID}.ome.tif"
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: expand_hc
          input: mask_hc
          output: mask_hc_expanded
          steps:
            - func: mask_dilate_by_disk
              params:
                radius: 3
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_mask_remove
              params:
                _mask: mask_hc_expanded
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
      process:
        name: 0d
        params:

    euchromatin_1d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        mask_hc:
          path: "results/segmentation/hc/{RoiUID}.ome.tif"
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: expand_hc
          input: mask_hc
          output: mask_hc_expanded
          steps:
            - func: mask_dilate_by_disk
              params:
                radius: 3
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_mask_remove
              params:
                _mask: mask_hc_expanded
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
      process:
        name: 1d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25
    
    euchromatin_2d:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        mask_hc:
          path: "results/segmentation/hc/{RoiUID}.ome.tif"
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: expand_hc
          input: mask_hc
          output: mask_hc_expanded
          steps:
            - func: mask_dilate_by_disk
              params:
                radius: 3
        - name: filter_piv
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_mask_remove
              params:
                _mask: mask_hc_expanded
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
      process:
        name: 2d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25

    # regular over all nucleoplasmic pixels, using fluctuation instead of
    # raw displacement
    normal_0d_fluctuation:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv_convert_to_fluctuation
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
            - func: piv_to_fluctuation
              params:
                components: [ 'u', 'v' ]
      process:
        name: 0d
        params:

    # 1D intensity-binned, using fluctuation instead of raw displacement
    eachlevel_1d_fluctuation:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv_convert_to_fluctuation
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: [ 'u', 'v' ]
                max: 4
            - func: piv_to_fluctuation
              params:
                components: [ 'u', 'v' ]
      process:
        name: 1d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25

    # 2D intensity-binned. using fluctuation instead of raw displacement
    eachlevel_2d_fluctuation:
      data: # relative to project dir
        piv:
          path:
          type: piv
        mask_nuc:
          path:
          type: mask
        mask_np:
          path:
          type: mask
        image:
          path: "results/image_normalized/by_nucleus/multi_ch/{RoiUID}.ome.tif"
          type: image
      preprocess:
        - name: blur_image
          input: image
          output: image_final
          steps:
            - func: image_blur
              params:
                sigma: 0.5
        - name: shrink_nucleus
          input: mask_nuc
          output: mask_nuc_inner
          steps:
            - func: mask_erode_by_disk
              params:
                radius: 15
        - name: filter_piv_convert_to_fluctuation
          input: piv
          output: piv_final
          steps:
            - func: piv_mask_filter
              params:
                _mask: mask_nuc_inner
            - func: piv_mask_filter
              params:
                _mask: mask_np
            - func: piv_global_filter
              params:
                components: ['u', 'v']
                max: 4
            - func: piv_to_fluctuation
              params:
                components: ['u', 'v']
      process:
        name: 2d
        params:
          bins_args:
            - -3.5
            - 3.5
            - 0.25


# ----------------------------------------------------------------
# MSND post-analysis
#
# Make sure to also update configs for each of the protocol used
# in msnd block
# ----------------------------------------------------------------
msnd_post:
  channel:
    - miRFP670
    - sfGFP
  groupby:
    normal_0d:
    eachlevel_1d:
      - channel
      - level
    eachlevel_2d:
    normal_0d_fluctuation:
    eachlevel_1d_fluctuation:
      - channel
      - level
    eachlevel_2d_fluctuation:
    euchromatin_0d:
    euchromatin_1d:
      - channel
      - level
    euchromatin_2d:
  min_size:
    normal_0d: 10
    eachlevel_1d: 30
    eachlevel_2d: 10
    normal_0d_fluctuation: 10
    eachlevel_1d_fluctuation: 30
    eachlevel_2d_fluctuation: 10
    euchromatin_0d: 10
    euchromatin_1d: 30
    euchromatin_2d: 10
  min_npoints:
    normal_0d: #leave blank
    eachlevel_1d: 5
    eachlevel_2d: 5
    normal_0d_fluctuation: #leave blank
    eachlevel_1d_fluctuation: 5
    eachlevel_2d_fluctuation: 5
    euchromatin_0d: #leave blank
    euchromatin_1d: 5
    euchromatin_2d: 5