name: "process_integrate_annotate"
namespace: "single_cell"
description: |
  A pipeline to process, integrate and annotate single cell (multi-)omics data.
  Available integration methods:
  - Harmony
  - scVI
  Available annotation methods:
  - CellTypist
  - scANVI (with scArches)

authors:
  - __merge__: /src/authors/dorien_roosen.yaml
    roles: [ author, maintainer ]
  - __merge__: /src/authors/weiwei_schultz.yaml
    roles: [ contributor ]

argument_groups:
  - name: Input (query) data arguments
    description: The input query dataset(s) to be annotated
    arguments:
      - name: "--id"
        required: true
        type: string
        description: ID of the sample.
        example: foo
      - name: "--input"
        required: true
        type: file
        description: Input query dataset(s) to be annotated
        example: input.h5mu
      - name: "--modality"
        default: "rna"
        type: string
        description: Modality to be processed. Should match the modality in the --reference dataset, if provided.
      - name: "--input_layer"
        type: string
        description: "The layer in the input data containing the raw counts, if .X is not to be used."
        required: false
      - name: "--input_var_gene_names"
        type: string
        required: false
        description: |
          The name of the adata var column containing gene names; when no gene_name_layer is provided, the var index will be used.
      - name: "--input_reference_gene_overlap"
        type: integer
        default: 100
        min: 1
        description: | 
          The minimum number of genes present in both the reference and query datasets.

  - name: Reference data arguments
    description: Dataset to be used as a reference for label transfer and to train annotation algorithms on.
    arguments:
      - name: "--reference"
        type: file
        required: false
        example: reference.h5mu
        description: |
          The reference dataset in .h5mu format to be used as a reference mapper and to train annotation algorithms on.
      - name: "--reference_layer_raw_counts"
        type: string
        description: "The layer in the reference dataset containing the raw counts, if .X is not to be used."
        required: false
      - name: "--reference_layer_lognormalized_counts"
        type: string
        default: log_normalized
        description: "The layer in the reference dataset containing the log-normalized counts, if .X is not to be used."
      - name: "--reference_var_gene_names"
        type: string
        required: false
        description: |
          The name of the adata .var column containing gene names if the .var index is not to be used.
      - name: "--reference_obs_batch"
        type: string
        required: false 
        description: |
          The .obs column of the reference dataset containing the batch information.
      - name: "--reference_obs_label"
        type: string
        example: cell_type
        required: false
        description: The `.obs` key of the target labels to tranfer.
      - name: "--reference_obs_label_unlabeled_category"
        type: string
        default: "Unkown"
        description: "Value in the --reference_obs_label field that indicates unlabeled observations"
      - name: "--reference_var_input"
        type: string
        required: false
        description: |
          .var column containing highly variable genes. By default, do not subset genes.


  - name: Methods
    description: The available annotation and integration methods to integrate and/or annotate the query dataset(s) with.
    arguments:
      - name: "--integration_methods"
        type: string
        multiple: true
        required: false
        choices: [harmony, scvi]
        example: harmony;scvi
        description: Integration methods to be executed.
      - name: "--annotation_methods"
        type: string
        multiple: true
        required: false
        choices: [celltypist, scanvi_scarches]
        example: celltypist;scanvi_scarches
        description: Annotation methods to be executed.
        
  - name: "Pre-processing options: RNA filtering"
    description: Pre-processing options for filtering RNA data
    arguments:
      - name: "--rna_min_counts"
        example: 200
        type: integer
        description: Minimum number of counts captured per cell.
      - name: "--rna_max_counts"
        example: 5000000
        type: integer
        description: Maximum number of counts captured per cell.
      - name: "--rna_min_genes_per_cell"
        type: integer
        example: 200
        description: Minimum of non-zero values per cell.
      - name: "--rna_max_genes_per_cell"
        example: 1500000
        type: integer
        description: Maximum of non-zero values per cell.
      - name: "--rna_min_cells_per_gene"
        example: 3
        type: integer
        description: Minimum of non-zero values per gene.
      - name: "--rna_min_fraction_mito"
        example: 0
        type: double
        description: Minimum fraction of UMIs that are mitochondrial.
      - name: "--rna_max_fraction_mito"
        type: double
        example: 0.2
        description: Maximum fraction of UMIs that are mitochondrial.

  - name: "Pre-processing options: Highly variable features detection"
    description: Pre-processing options for detecting highly variable features
    arguments:
      - name: "--n_hvg"
        type: integer
        description: |
          Number of highly-variable features to keep. 
          Only relevant if HVG need to be calculated across query and reference datasets (e.g. for --annotation_methods scvi_knn and harmony_knn). 
          For reference mapping-based methods, the HVG's specified in --reference_var_input will be used.
        default: 2000
  
  - name: "Pre-processing options: Mitochondrial & Ribosomal Gene Detection"
    description: Pre-processing options for detecting mitochondrial genes
    arguments:
      - name: "--var_name_mitochondrial_genes"
        type: string
        required: false
        description: |
          In which .var slot to store a boolean array corresponding the mitochondrial genes.
      - name: "--var_name_ribosomal_genes"
        type: string
        required: false
        description: |
          In which .var slot to store a boolean array corresponding the ribosomal genes.
      - name: "--obs_name_mitochondrial_fraction"
        type: string
        required: false
        description: |
          When specified, write the fraction of counts originating from mitochondrial genes 
          (based on --mitochondrial_gene_regex) to an .obs column with the specified name.
          Requires --var_name_mitochondrial_genes.
      - name: "--obs_name_ribosomal_fraction"
        type: string
        required: false
        description: |
          When specified, write the fraction of counts originating from ribosomal genes 
          (based on --ribosomal_gene_regex) to an .obs column with the specified name.
          Requires --var_name_ribosomal_genes.
      - name: --mitochondrial_gene_regex
        type: string
        description: |
          Regex string that identifies mitochondrial genes from --var_gene_names.
          By default will detect human and mouse mitochondrial genes from a gene symbol.
        required: false
        default: "^[mM][tT]-"
      - name: --ribosomal_gene_regex
        type: string
        description: |
          Regex string that identifies ribosomal genes from --var_gene_names.
          By default will detect human and mouse ribosomal genes from a gene symbol.
        required: false
        default: "^[Mm]?[Rr][Pp][LlSs]"

  - name: "Pre-processing options: QC metrics calculation options"
    description: Pre-processing options for calculating QC metrics
    arguments:
      - name: "--var_qc_metrics"
        description: |
          Keys to select a boolean (containing only True or False) column from .var.
          For each cell, calculate the proportion of total values for genes which are labeled 'True', 
          compared to the total sum of the values for all genes. Defaults to the combined values specified for
          --var_name_mitochondrial_genes and --highly_variable_features_var_output.
        type: string
        multiple: True
        multiple_sep: ','
        required: false
        example: "ercc,highly_variable"

  - name: Harmony integration options
    description: Specifications for harmony integration.
    arguments:
      - name: "--harmony_theta"
        type: double
        description: |
          Diversity clustering penalty parameter. Specify for each variable in group.by.vars. 
          theta=0 does not encourage any diversity. Larger values of theta
          result in more diverse clusters."
        default: 2
        example: [0, 1, 2]
        multiple: true
      - name: "--harmony_obs_covariates"
        type: string
        description: "The .obs field(s) that define the covariate(s) to regress out."
        example: ["batch", "sample"]
        required: true
        multiple: true
        default: "sample_id"

  - name: scVI, scANVI and scArches training options
    # TODO - possibly provide separate training options for scVI, scANVI and scArches
    description: Training arguments for scVI, scANVI and scArches. Relevant for --annotation_methods 'scvi_knn' and 'scanvi_scarches'.
    arguments:
      - name: "--early_stopping"
        required: false
        type: boolean
        description: "Whether to perform early stopping with respect to the validation set."
      - name: "--early_stopping_monitor"
        choices: ["elbo_validation", "reconstruction_loss_validation", "kl_local_validation"]
        default: "elbo_validation"
        type: string
        description: "Metric logged during validation set epoch."
      - name: "--early_stopping_patience"
        type: integer
        min: 1
        default: 45
        description: "Number of validation epochs with no improvement after which training will be stopped."
      - name: "--early_stopping_min_delta"
        min: 0
        type: double
        default: 0.0
        description: "Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute change of less than min_delta, will count as no improvement."
      - name: "--max_epochs"
        type: integer
        description: "Number of passes through the dataset, defaults to (20000 / number of cells) * 400 or 400; whichever is smallest."
        required: false
      - name: "--reduce_lr_on_plateau"
        description: "Whether to monitor validation loss and reduce learning rate when validation set `lr_scheduler_metric` plateaus."
        type: boolean
        default: True
      - name: "--lr_factor"
        description: "Factor to reduce learning rate."
        type: double
        default: 0.6
        min: 0
      - name: "--lr_patience"
        description: "Number of epochs with no improvement after which learning rate will be reduced."
        type: double
        default: 30
        min: 0

  - name: CellTypist reference model
    description: The CellTypist reference model to use for annotation. If not provided, the reference dataset will be used for model training.
    arguments:
      - name: "--celltypist_model"
        type: file
        description: "Pretrained model in pkl format. If not provided, the model will be trained on the reference data and --reference should be provided."
        required: false
        example: pretrained_model.pkl

  - name: CellTypist annotation options
    description: Specifications for CellTypist annotation.
    arguments:
      - name: "--celltypist_feature_selection"
        type: boolean
        description: "Whether to perform feature selection."
        default: false
      - name: "--celltypist_majority_voting"
        type: boolean
        description: "Whether to refine the predicted labels by running the majority voting classifier after over-clustering."
        default: false
      - name: "--celltypist_C"
        type: double
        description: "Inverse of regularization strength in logistic regression."
        default: 1.0
      - name: "--celltypist_max_iter"
        type: integer
        description: "Maximum number of iterations before reaching the minimum of the cost function."
        default: 1000
      - name: "--celltypist_use_SGD"
        type: boolean_true
        description: "Whether to use the stochastic gradient descent algorithm."
      - name: "--celltypist_min_prop"
        type: double
        description: |
          "For the dominant cell type within a subcluster, the minimum proportion of cells required to 
          support naming of the subcluster by this cell type. Ignored if majority_voting is set to False. 
          Subcluster that fails to pass this proportion threshold will be assigned 'Heterogeneous'."
        default: 0

  - name: Clustering options
    description: Arguments for Leiden clustering. Only relevant for --annotation_methods `scvi_knn`, `scanvi_scarches` and `harmony_knn`.
    arguments:
      - name: "--leiden_resolution"
        type: double
        description: Control the coarseness of the clustering. Higher values lead to more clusters.
        default: [1]
        multiple: true

  - name: Neighbor classifier arguments
    description: Arguments related to calculating the n nearest neighbors. Only relevant for --annotation_methods `scvi_knn`, `scanvi_scarches` and `harmony_knn`.
    arguments:
      - name: "--knn_weights"
        type: string
        default: "uniform"
        choices: ["uniform", "distance"]
        description: |
          Weight function used in prediction. Possible values are:
          `uniform` (all points in each neighborhood are weighted equally) or 
          `distance` (weight points by the inverse of their distance)
      - name: "--knn_n_neighbors"
        type: integer
        default: 15
        min: 5
        required: false
        description: |
          The number of neighbors to use in k-neighbor graph structure used for fast approximate nearest neighbor search with PyNNDescent. 
          Larger values will result in more accurate search results at the cost of computation time.

  - name: Outputs
    description: The output file to write the annotated dataset to.
    arguments:
    - name: "--output"
      type: file
      direction: output
      required: true
      description: |
        The output file.
      example: output.h5mu

dependencies:
  - name: workflows/multiomics/process_samples
    alias: process_samples_workflow
    repository: openpipeline
  - name: annotate/celltypist
    repository: openpipeline
    alias: celltypist_annotation
  - name: workflows/annotation/scanvi_scarches
    repository: openpipeline
    alias: scanvi_scarches_annotation
  - name: workflows/integration/harmony_leiden
    repository: openpipeline
    alias: harmony_integration
  - name: workflows/integration/scvi_leiden
    repository: openpipeline
    alias: scvi_integration

resources:
  - type: nextflow_script
    path: main.nf
    entrypoint: run_wf

test_resources:
  - type: nextflow_script
    path: test.nf
    entrypoint: test_wf
  - path: /resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_mms.h5mu
  - path: /resources_test/annotation_test_data/TS_Blood_filtered.h5mu
  - path: /resources_test/annotation_test_data/celltypist_model_Immune_All_Low.pkl

runners:
  - type: nextflow