From e3029375b3f7b635834c86ed3443a8a0ae271ed7 Mon Sep 17 00:00:00 2001 From: CI Date: Mon, 25 Aug 2025 12:10:43 +0000 Subject: [PATCH] Build branch v0.1 with version v0.1 (420b610) Build pipeline: openpipelines-bio.openpipeline-spatial.v0.1-rwpq2 Source commit: https://github.com/openpipelines-bio/openpipeline_spatial/commit/420b610b045c138f07475a1af39529f233d9af5a Source message: Merge pull request #18 from openpipelines-bio/update-op-dependencies Update OP dependency version --- CHANGELOG.md | 2 +- _viash.yaml | 1 - .../spatial_process_samples/config.vsh.yaml | 8 +- .../spatial_process_samples/main.nf | 2 +- .../cluster/leiden/nextflow_params.yaml | 12 - .../concatenate_h5mu/nextflow_params.yaml | 13 - .../dataflow/merge/nextflow_params.yaml | 8 - .../split_modalities/nextflow_labels.config | 66 - .../split_modalities/nextflow_params.yaml | 9 - .../dimred/pca/nextflow_labels.config | 66 - .../nextflow/dimred/pca/nextflow_params.yaml | 16 - .../dimred/umap/nextflow_labels.config | 66 - .../nextflow/dimred/umap/nextflow_params.yaml | 23 - .../nextflow_labels.config | 66 - .../nextflow_params.yaml | 22 - .../delimit_fraction/nextflow_labels.config | 66 - .../delimit_fraction/nextflow_params.yaml | 17 - .../filter/do_filter/nextflow_labels.config | 66 - .../filter/do_filter/nextflow_params.yaml | 11 - .../filter_with_counts/nextflow_labels.config | 66 - .../filter_with_counts/nextflow_params.yaml | 22 - .../nextflow_labels.config | 66 - .../filter_with_scrublet/nextflow_params.yaml | 23 - .../metadata/add_id/nextflow_labels.config | 66 - .../metadata/add_id/nextflow_params.yaml | 11 - .../grep_annotation_column/.config.vsh.yaml | 320 -- .../metadata/grep_annotation_column/main.nf | 4123 ---------------- .../grep_annotation_column/nextflow.config | 126 - .../nextflow_labels.config | 66 - .../nextflow_params.yaml | 21 - .../nextflow_schema.json | 131 - .../move_obsm_to_obs/nextflow_labels.config | 66 - .../move_obsm_to_obs/nextflow_params.yaml | 14 - .../find_neighbors/nextflow_labels.config | 66 - .../find_neighbors/nextflow_params.yaml | 16 - .../qc/calculate_qc_metrics/.config.vsh.yaml | 380 -- .../nextflow/qc/calculate_qc_metrics/main.nf | 4220 ----------------- .../qc/calculate_qc_metrics/nextflow.config | 126 - .../nextflow_labels.config | 66 - .../calculate_qc_metrics/nextflow_params.yaml | 27 - .../calculate_qc_metrics/nextflow_schema.json | 170 - .../transform/clr/nextflow_labels.config | 66 - .../transform/clr/nextflow_params.yaml | 12 - .../delete_layer/nextflow_labels.config | 66 - .../delete_layer/nextflow_params.yaml | 11 - .../transform/log1p/nextflow_labels.config | 66 - .../transform/log1p/nextflow_params.yaml | 12 - .../normalize_total/compress_h5mu.py | 87 - .../normalize_total/nextflow_labels.config | 66 - .../normalize_total/nextflow_params.yaml | 13 - .../transform/normalize_total/setup_logger.py | 12 - .../nextflow/transform/scale/compress_h5mu.py | 87 - .../transform/scale/nextflow_labels.config | 66 - .../transform/scale/nextflow_params.yaml | 13 - .../nextflow/transform/scale/setup_logger.py | 12 - .../gdo_singlesample/nextflow_labels.config | 66 - .../gdo/gdo_singlesample/nextflow_params.yaml | 20 - .../nextflow_labels.config | 66 - .../nextflow_params.yaml | 29 - .../nextflow_labels.config | 66 - .../nextflow_params.yaml | 25 - .../process_batches/nextflow_labels.config | 66 - .../process_batches/nextflow_params.yaml | 38 - .../process_batches/utils/labels.config | 66 - .../process_samples/nextflow_labels.config | 66 - .../process_samples/nextflow_params.yaml | 79 - .../process_samples/utils/labels.config | 66 - .../prot_multisample/nextflow_labels.config | 66 - .../prot_multisample/nextflow_params.yaml | 26 - .../prot/prot_multisample/utils/labels.config | 66 - .../prot_singlesample/nextflow_labels.config | 66 - .../prot_singlesample/nextflow_params.yaml | 20 - .../prot_singlesample/utils/labels.config | 66 - .../nextflow/workflows/qc/qc/.config.vsh.yaml | 406 -- .../nextflow/workflows/qc/qc/main.nf | 3758 --------------- .../nextflow/workflows/qc/qc/nextflow.config | 126 - .../workflows/qc/qc/nextflow_labels.config | 66 - .../workflows/qc/qc/nextflow_params.yaml | 33 - .../workflows/qc/qc/nextflow_schema.json | 204 - .../workflows/qc/qc/utils/labels.config | 66 - .../rna_multisample/nextflow_labels.config | 66 - .../rna/rna_multisample/nextflow_params.yaml | 36 - .../rna/rna_multisample/utils/labels.config | 66 - .../rna_singlesample/nextflow_labels.config | 66 - .../rna/rna_singlesample/nextflow_params.yaml | 34 - .../utils/errorstrat_ignore.config | 1 - .../utils/integration_tests.config | 36 - .../rna/rna_singlesample/utils/labels.config | 66 - .../rna_singlesample/utils/labels_ci.config | 33 - .../split_modalities/.config.vsh.yaml | 20 +- .../multiomics/split_modalities/main.nf | 29 +- .../split_modalities/nextflow.config | 2 +- .../split_modalities}/nextflow_labels.config | 34 +- .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../split_modalities/utils/labels.config | 34 +- .../split_modalities/utils/labels_ci.config | 0 .../nextflow/cluster/leiden/.config.vsh.yaml | 25 +- .../nextflow/cluster/leiden/compress_h5mu.py | 0 .../v3.0.0}/nextflow/cluster/leiden/main.nf | 45 +- .../nextflow/cluster/leiden/nextflow.config | 2 +- .../cluster/leiden/nextflow_labels.config | 34 +- .../cluster/leiden/nextflow_schema.json | 14 - .../nextflow/cluster/leiden/setup_logger.py | 0 .../concatenate_h5mu/.config.vsh.yaml | 25 +- .../concatenate_h5mu/compress_h5mu.py | 0 .../dataflow/concatenate_h5mu/main.nf | 45 +- .../dataflow/concatenate_h5mu/nextflow.config | 2 +- .../concatenate_h5mu/nextflow_labels.config | 34 +- .../concatenate_h5mu/nextflow_schema.json | 14 - .../dataflow/concatenate_h5mu/setup_logger.py | 0 .../nextflow/dataflow/merge/.config.vsh.yaml | 25 +- .../v3.0.0}/nextflow/dataflow/merge/main.nf | 45 +- .../nextflow/dataflow/merge/nextflow.config | 2 +- .../dataflow/merge}/nextflow_labels.config | 34 +- .../dataflow/merge/nextflow_schema.json | 14 - .../nextflow/dataflow/merge/setup_logger.py | 0 .../split_modalities/.config.vsh.yaml | 25 +- .../dataflow/split_modalities/main.nf | 45 +- .../dataflow/split_modalities/nextflow.config | 2 +- .../split_modalities/nextflow_labels.config | 48 + .../split_modalities/nextflow_schema.json | 14 - .../dataflow/split_modalities/setup_logger.py | 0 .../nextflow/dimred/pca/.config.vsh.yaml | 25 +- .../nextflow/dimred/pca/compress_h5mu.py | 0 .../v3.0.0}/nextflow/dimred/pca/main.nf | 45 +- .../nextflow/dimred/pca/nextflow.config | 2 +- .../dimred/pca/nextflow_labels.config | 48 + .../nextflow/dimred/pca/nextflow_schema.json | 14 - .../nextflow/dimred/pca/setup_logger.py | 0 .../nextflow/dimred/umap/.config.vsh.yaml | 25 +- .../nextflow/dimred/umap/compress_h5mu.py | 0 .../v3.0.0}/nextflow/dimred/umap/main.nf | 45 +- .../nextflow/dimred/umap/nextflow.config | 2 +- .../dimred/umap/nextflow_labels.config | 48 + .../nextflow/dimred/umap/nextflow_schema.json | 14 - .../nextflow/dimred/umap/setup_logger.py | 0 .../.config.vsh.yaml | 25 +- .../compress_h5mu.py | 0 .../highly_variable_features_scanpy/main.nf | 45 +- .../nextflow.config | 2 +- .../nextflow_labels.config | 48 + .../nextflow_schema.json | 14 - .../setup_logger.py | 0 .../subset_vars.py | 0 .../filter/delimit_fraction/.config.vsh.yaml | 25 +- .../filter/delimit_fraction/compress_h5mu.py | 0 .../nextflow/filter/delimit_fraction/main.nf | 45 +- .../filter/delimit_fraction/nextflow.config | 2 +- .../delimit_fraction/nextflow_labels.config | 48 + .../delimit_fraction/nextflow_schema.json | 14 - .../filter/delimit_fraction/setup_logger.py | 0 .../filter/do_filter/.config.vsh.yaml | 25 +- .../filter/do_filter/compress_h5mu.py | 0 .../v3.0.0}/nextflow/filter/do_filter/main.nf | 45 +- .../nextflow/filter/do_filter/nextflow.config | 2 +- .../filter/do_filter/nextflow_labels.config | 48 + .../filter/do_filter/nextflow_schema.json | 14 - .../nextflow/filter/do_filter/setup_logger.py | 0 .../filter_with_counts/.config.vsh.yaml | 25 +- .../filter_with_counts/compress_h5mu.py | 0 .../filter/filter_with_counts/main.nf | 45 +- .../filter/filter_with_counts/nextflow.config | 2 +- .../filter_with_counts/nextflow_labels.config | 48 + .../filter_with_counts/nextflow_schema.json | 14 - .../filter/filter_with_counts/setup_logger.py | 0 .../filter_with_scrublet/.config.vsh.yaml | 25 +- .../filter_with_scrublet/compress_h5mu.py | 0 .../filter/filter_with_scrublet/main.nf | 45 +- .../filter_with_scrublet/nextflow.config | 2 +- .../nextflow_labels.config | 48 + .../filter_with_scrublet/nextflow_schema.json | 14 - .../filter_with_scrublet/setup_logger.py | 0 .../nextflow/metadata/add_id/.config.vsh.yaml | 25 +- .../v3.0.0}/nextflow/metadata/add_id/main.nf | 45 +- .../nextflow/metadata/add_id/nextflow.config | 2 +- .../metadata/add_id/nextflow_labels.config | 48 + .../metadata/add_id/nextflow_schema.json | 14 - .../nextflow/metadata/add_id/setup_logger.py | 0 .../move_obsm_to_obs/.config.vsh.yaml | 25 +- .../move_obsm_to_obs}/compress_h5mu.py | 0 .../metadata/move_obsm_to_obs/main.nf | 45 +- .../metadata/move_obsm_to_obs/nextflow.config | 2 +- .../move_obsm_to_obs/nextflow_labels.config | 48 + .../move_obsm_to_obs/nextflow_schema.json | 14 - .../move_obsm_to_obs}/setup_logger.py | 0 .../neighbors/find_neighbors/.config.vsh.yaml | 25 +- .../find_neighbors}/compress_h5mu.py | 0 .../nextflow/neighbors/find_neighbors/main.nf | 45 +- .../neighbors/find_neighbors/nextflow.config | 2 +- .../find_neighbors/nextflow_labels.config | 48 + .../find_neighbors/nextflow_schema.json | 14 - .../neighbors/find_neighbors}/setup_logger.py | 0 .../nextflow/transform/clr/.config.vsh.yaml | 25 +- .../nextflow/transform/clr}/compress_h5mu.py | 0 .../v3.0.0}/nextflow/transform/clr/main.nf | 45 +- .../nextflow/transform/clr/nextflow.config | 2 +- .../transform/clr/nextflow_labels.config | 48 + .../transform/clr/nextflow_schema.json | 14 - .../transform/delete_layer/.config.vsh.yaml | 25 +- .../transform/delete_layer}/compress_h5mu.py | 0 .../nextflow/transform/delete_layer/main.nf | 45 +- .../transform/delete_layer/nextflow.config | 2 +- .../delete_layer/nextflow_labels.config | 48 + .../delete_layer/nextflow_schema.json | 14 - .../transform/delete_layer}/setup_logger.py | 0 .../nextflow/transform/log1p/.config.vsh.yaml | 25 +- .../transform/log1p}/compress_h5mu.py | 0 .../v3.0.0}/nextflow/transform/log1p/main.nf | 45 +- .../nextflow/transform/log1p/nextflow.config | 2 +- .../transform/log1p/nextflow_labels.config | 48 + .../transform/log1p/nextflow_schema.json | 14 - .../nextflow/transform/log1p}/setup_logger.py | 0 .../normalize_total/.config.vsh.yaml | 25 +- .../normalize_total}/compress_h5mu.py | 0 .../transform/normalize_total/main.nf | 45 +- .../transform/normalize_total/nextflow.config | 2 +- .../normalize_total/nextflow_labels.config | 48 + .../normalize_total/nextflow_schema.json | 14 - .../normalize_total}/setup_logger.py | 0 .../nextflow/transform/scale/.config.vsh.yaml | 25 +- .../transform/scale}/compress_h5mu.py | 0 .../v3.0.0}/nextflow/transform/scale/main.nf | 45 +- .../nextflow/transform/scale/nextflow.config | 2 +- .../transform/scale/nextflow_labels.config | 48 + .../transform/scale/nextflow_schema.json | 14 - .../nextflow/transform/scale}/setup_logger.py | 0 .../gdo/gdo_singlesample/.config.vsh.yaml | 20 +- .../workflows/gdo/gdo_singlesample/main.nf | 29 +- .../gdo/gdo_singlesample/nextflow.config | 2 +- .../gdo_singlesample/nextflow_labels.config | 48 + .../gdo/gdo_singlesample/nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../gdo/gdo_singlesample/utils/labels.config | 34 +- .../gdo_singlesample/utils/labels_ci.config | 0 .../dimensionality_reduction/.config.vsh.yaml | 20 +- .../dimensionality_reduction/main.nf | 29 +- .../dimensionality_reduction/nextflow.config | 2 +- .../nextflow_labels.config | 48 + .../nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../utils/labels.config | 34 +- .../utils/labels_ci.config | 0 .../neighbors_leiden_umap/.config.vsh.yaml | 20 +- .../multiomics/neighbors_leiden_umap/main.nf | 29 +- .../neighbors_leiden_umap/nextflow.config | 2 +- .../nextflow_labels.config | 48 + .../nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../neighbors_leiden_umap/utils/labels.config | 34 +- .../utils/labels_ci.config | 0 .../process_batches/.config.vsh.yaml | 20 +- .../multiomics/process_batches/main.nf | 29 +- .../process_batches/nextflow.config | 2 +- .../process_batches/nextflow_labels.config | 48 + .../process_batches/nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../process_batches/utils/labels.config | 48 + .../process_batches/utils/labels_ci.config | 0 .../process_samples/.config.vsh.yaml | 24 +- .../multiomics/process_samples/main.nf | 35 +- .../process_samples/nextflow.config | 2 +- .../process_samples/nextflow_labels.config | 48 + .../process_samples/nextflow_schema.json | 18 +- .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../process_samples/utils/labels.config | 48 + .../process_samples/utils/labels_ci.config | 0 .../prot/prot_multisample/.config.vsh.yaml | 20 +- .../workflows/prot/prot_multisample/main.nf | 29 +- .../prot/prot_multisample/nextflow.config | 2 +- .../prot_multisample/nextflow_labels.config | 48 + .../prot_multisample/nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../prot/prot_multisample/utils/labels.config | 48 + .../prot_multisample/utils/labels_ci.config | 0 .../prot/prot_singlesample/.config.vsh.yaml | 20 +- .../workflows/prot/prot_singlesample/main.nf | 29 +- .../prot/prot_singlesample/nextflow.config | 2 +- .../prot_singlesample/nextflow_labels.config | 48 + .../prot_singlesample/nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../prot_singlesample/utils/labels.config | 48 + .../prot_singlesample/utils/labels_ci.config | 0 .../rna/rna_multisample/.config.vsh.yaml | 20 +- .../workflows/rna/rna_multisample/main.nf | 29 +- .../rna/rna_multisample/nextflow.config | 2 +- .../rna_multisample/nextflow_labels.config | 48 + .../rna/rna_multisample/nextflow_schema.json | 14 - .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../rna/rna_multisample/utils/labels.config | 48 + .../rna_multisample}/utils/labels_ci.config | 0 .../rna/rna_singlesample/.config.vsh.yaml | 24 +- .../workflows/rna/rna_singlesample/main.nf | 52 +- .../rna/rna_singlesample/nextflow.config | 2 +- .../rna_singlesample/nextflow_labels.config | 48 + .../rna/rna_singlesample/nextflow_schema.json | 18 +- .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../rna/rna_singlesample/utils/labels.config | 48 + .../rna_singlesample}/utils/labels_ci.config | 0 .../from_cells2stats_to_h5mu/.config.vsh.yaml | 10 +- .../from_cells2stats_to_h5mu | 14 +- .../from_cosmx_to_h5mu/.config.vsh.yaml | 10 +- .../from_cosmx_to_h5mu/from_cosmx_to_h5mu | 14 +- .../.config.vsh.yaml | 10 +- .../from_cosmx_to_spatialexperiment | 14 +- .../.config.vsh.yaml | 10 +- .../from_h5mu_to_spatialexperiment | 14 +- .../from_spatialdata_to_h5mu/.config.vsh.yaml | 10 +- .../from_spatialdata_to_h5mu | 14 +- .../from_xenium_to_h5mu/.config.vsh.yaml | 10 +- .../from_xenium_to_h5mu/from_xenium_to_h5mu | 14 +- .../.config.vsh.yaml | 10 +- .../from_xenium_to_spatialdata | 14 +- .../.config.vsh.yaml | 10 +- .../from_xenium_to_spatialexperiment | 14 +- .../filter/subset_cosmx/.config.vsh.yaml | 10 +- .../filter/subset_cosmx/subset_cosmx | 14 +- .../spaceranger_count/.config.vsh.yaml | 10 +- .../spaceranger_count/spaceranger_count | 14 +- .../from_cells2stats_to_h5mu/.config.vsh.yaml | 10 +- .../convert/from_cells2stats_to_h5mu/main.nf | 14 +- .../from_cells2stats_to_h5mu/nextflow.config | 2 +- .../from_cosmx_to_h5mu/.config.vsh.yaml | 10 +- .../convert/from_cosmx_to_h5mu/main.nf | 14 +- .../from_cosmx_to_h5mu/nextflow.config | 2 +- .../.config.vsh.yaml | 10 +- .../from_cosmx_to_spatialexperiment/main.nf | 14 +- .../nextflow.config | 2 +- .../.config.vsh.yaml | 10 +- .../from_h5mu_to_spatialexperiment/main.nf | 14 +- .../nextflow.config | 2 +- .../from_spatialdata_to_h5mu/.config.vsh.yaml | 10 +- .../convert/from_spatialdata_to_h5mu/main.nf | 14 +- .../from_spatialdata_to_h5mu/nextflow.config | 2 +- .../from_xenium_to_h5mu/.config.vsh.yaml | 10 +- .../convert/from_xenium_to_h5mu/main.nf | 14 +- .../from_xenium_to_h5mu/nextflow.config | 2 +- .../.config.vsh.yaml | 10 +- .../from_xenium_to_spatialdata/main.nf | 14 +- .../nextflow.config | 2 +- .../.config.vsh.yaml | 10 +- .../from_xenium_to_spatialexperiment/main.nf | 14 +- .../nextflow.config | 2 +- .../filter/subset_cosmx/.config.vsh.yaml | 10 +- target/nextflow/filter/subset_cosmx/main.nf | 14 +- .../filter/subset_cosmx/nextflow.config | 2 +- .../spaceranger_count/.config.vsh.yaml | 10 +- .../mapping/spaceranger_count/main.nf | 14 +- .../mapping/spaceranger_count/nextflow.config | 2 +- .../spatial_process_samples/.config.vsh.yaml | 20 +- .../spatial_process_samples/main.nf | 26 +- .../spatial_process_samples/nextflow.config | 2 +- .../workflows/qc/spatial_qc/.config.vsh.yaml | 8 +- .../nextflow/workflows/qc/spatial_qc/main.nf | 10 +- .../workflows/qc/spatial_qc/nextflow.config | 2 +- 364 files changed, 3095 insertions(+), 18895 deletions(-) delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/.config.vsh.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/main.nf delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_schema.json delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/main.nf delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_schema.json delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/compress_h5mu.py delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/setup_logger.py delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/compress_h5mu.py delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/setup_logger.py delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/.config.vsh.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/main.nf delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_schema.json delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_params.yaml delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels.config delete mode 100644 target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge => vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities}/nextflow_labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/.config.vsh.yaml (93%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/nextflow.config (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/nextflow_labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/nextflow_schema.json (84%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/cluster/leiden/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/nextflow.config (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json (87%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/concatenate_h5mu/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/merge/.config.vsh.yaml (91%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/merge/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/merge/nextflow.config (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities => vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge}/nextflow_labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/merge/nextflow_schema.json (74%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/merge/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/split_modalities/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/split_modalities/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/split_modalities/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/split_modalities/nextflow_schema.json (77%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dataflow/split_modalities/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/.config.vsh.yaml (93%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/nextflow_schema.json (85%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/pca/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/nextflow_schema.json (88%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/dimred/umap/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml (95%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json (90%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/.config.vsh.yaml (93%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/nextflow_schema.json (84%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/delimit_fraction/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/nextflow_schema.json (80%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/do_filter/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/nextflow_schema.json (87%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_counts/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/.config.vsh.yaml (95%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/nextflow_schema.json (90%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/filter/filter_with_scrublet/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/add_id/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/add_id/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/add_id/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/add_id/nextflow_schema.json (79%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/add_id/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column => vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/move_obsm_to_obs/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/move_obsm_to_obs/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json (79%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column => vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/neighbors/find_neighbors/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs => vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/neighbors/find_neighbors/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/neighbors/find_neighbors/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/neighbors/find_neighbors/nextflow_schema.json (88%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs => vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/clr/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/clr/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/clr/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/clr/nextflow_schema.json (80%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/delete_layer/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/delete_layer/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/delete_layer/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/delete_layer/nextflow_schema.json (79%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/log1p/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/log1p/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/log1p/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/log1p/nextflow_schema.json (79%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/normalize_total/.config.vsh.yaml (93%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/normalize_total/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/normalize_total/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/normalize_total/nextflow_schema.json (83%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/scale/.config.vsh.yaml (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale}/compress_h5mu.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/scale/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/scale/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/transform/scale/nextflow_schema.json (80%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p => vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale}/setup_logger.py (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml (93%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json (83%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json (88%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json (86%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config (55%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml (96%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/nextflow_schema.json (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml (97%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/nextflow_schema.json (95%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/nextflow_schema.json (89%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml (94%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json (82%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml (96%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_multisample/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_multisample/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_multisample/nextflow_schema.json (92%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample}/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample}/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample}/utils/labels_ci.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml (95%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_singlesample/main.nf (99%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_singlesample/nextflow.config (99%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build => vsh/vsh/openpipeline/v3.0.0}/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json (89%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample}/utils/errorstrat_ignore.config (100%) rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample}/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels.config rename target/dependencies/{github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample => vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample}/utils/labels_ci.config (100%) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a4468f..5bf9cb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -# openpipeline_spatial 0.1.0 +# openpipeline_spatial 0.0.0 ## NEW FUNCTIONALITY diff --git a/_viash.yaml b/_viash.yaml index 884cccf..504fcce 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -1,4 +1,3 @@ -version: v0.1.0 viash_version: 0.9.4 source: src diff --git a/src/workflows/multiomics/spatial_process_samples/config.vsh.yaml b/src/workflows/multiomics/spatial_process_samples/config.vsh.yaml index 27b1917..3a2ac50 100644 --- a/src/workflows/multiomics/spatial_process_samples/config.vsh.yaml +++ b/src/workflows/multiomics/spatial_process_samples/config.vsh.yaml @@ -295,13 +295,7 @@ argument_groups: dependencies: - name: workflows/multiomics/process_samples alias: spatial_sample_processing - repository: openpipeline_scrublet - -repositories: - - name: openpipeline_scrublet - repo: openpipelines-bio/openpipeline - type: github - tag: disable-scrublet_build + repository: openpipeline resources: - type: nextflow_script diff --git a/src/workflows/multiomics/spatial_process_samples/main.nf b/src/workflows/multiomics/spatial_process_samples/main.nf index 2c518b8..2a2c7fc 100644 --- a/src/workflows/multiomics/spatial_process_samples/main.nf +++ b/src/workflows/multiomics/spatial_process_samples/main.nf @@ -58,7 +58,7 @@ workflow run_wf { "output": state.workflow_output ]}, args: [ - "skip_scrublet_filtering": "true", + "skip_scrublet_doublet_detection": "true", ], toState: [ "output": "output" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_params.yaml deleted file mode 100644 index c2808f0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_params.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -obsp_connectivities: "connectivities" -# output: "$id.$key.output.h5mu" -obsm_name: "leiden" -resolution: # please fill in - example: [1.0] -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_params.yaml deleted file mode 100644 index eb31bb1..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_params.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Arguments -input: # please fill in - example: ["sample_paths"] -# modality: ["foo"] -# input_id: ["foo"] -# output: "$id.$key.output.h5mu" -obs_sample_name: "sample_id" -other_axis_mode: "move" -uns_merge_mode: "make_unique" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_params.yaml deleted file mode 100644 index 9192489..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_params.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# Arguments -input: # please fill in - example: ["sample_paths"] -# output: "output.h5mu" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_params.yaml deleted file mode 100644 index fd89c11..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_params.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# Arguments -input: # please fill in - example: "sample_path" -# output: "$id.$key.output" -# output_types: "$id.$key.output_types.csv" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_params.yaml deleted file mode 100644 index 0713cc7..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_params.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "foo" -# var_input: "filter_with_hvg" -# output: "$id.$key.output.h5mu" -obsm_output: "X_pca" -varm_output: "pca_loadings" -uns_output: "pca_variance" -# num_components: 25 -overwrite: false -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_params.yaml deleted file mode 100644 index fdca0ea..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_params.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Inputs -input: # please fill in - example: "input.h5mu" -modality: "rna" -uns_neighbors: "neighbors" - -# Outputs -# output: "$id.$key.output.h5mu" -obsm_output: "umap" -# output_compression: "gzip" - -# Arguments -min_dist: 0.5 -spread: 1.0 -num_components: 2 -# max_iter: 123 -alpha: 1.0 -gamma: 1.0 -negative_sample_rate: 5 -init_pos: "spectral" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_params.yaml deleted file mode 100644 index 5a1fd82..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_params.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "foo" -# var_input: "foo" -# output: "$id.$key.output.h5mu" -var_name_filter: "filter_with_hvg" -varm_name: "hvg" -flavor: "seurat" -# n_top_features: 123 -min_mean: 0.0125 -max_mean: 3.0 -min_disp: 0.5 -# max_disp: 123.0 -span: 0.3 -n_bins: 20 -# obs_batch_key: "foo" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_params.yaml deleted file mode 100644 index 9b7c699..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_params.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Inputs -input: # please fill in - example: "input.h5mu" -modality: "rna" -obs_fraction_column: # please fill in - example: "fraction_mitochondrial" - -# Outputs -# output: "$id.$key.output.h5mu" -obs_name_filter: # please fill in - example: "foo" -# output_compression: "gzip" - -# Arguments -min_fraction: 0.0 -max_fraction: 1.0 - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_params.yaml deleted file mode 100644 index a0d7704..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_params.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# obs_filter: ["filter_with_x"] -# var_filter: ["filter_with_x"] -# output: "$id.$key.output.h5mu" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_params.yaml deleted file mode 100644 index b825b43..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_params.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Inputs -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "raw_counts" - -# Outputs -# output: "$id.$key.output.h5mu" -do_subset: false -obs_name_filter: "filter_with_counts" -var_name_filter: "filter_with_counts" -# output_compression: "gzip" - -# Arguments -# min_counts: 200 -# max_counts: 5000000 -# min_genes_per_cell: 200 -# max_genes_per_cell: 1500000 -# min_cells_per_gene: 3 - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_params.yaml deleted file mode 100644 index 62dcafa..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_params.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "foo" -# output: "$id.$key.output.h5mu" -obs_name_filter: "filter_with_scrublet" -do_subset: false -obs_name_doublet_score: "scrublet_doublet_score" -# expected_doublet_rate: 123.0 -# stdev_doublet_rate: 123.0 -# n_neighbors: 123 -# sim_doublet_ratio: 123.0 -min_counts: 2 -min_cells: 3 -min_gene_variablity_percent: 85.0 -num_pca_components: 30 -distance_metric: "euclidean" -allow_automatic_threshold_detection_fail: false -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_params.yaml deleted file mode 100644 index 5d7ae9c..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_params.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Arguments -input: # please fill in - example: "sample_path" -input_id: # please fill in - example: "foo" -obs_output: "sample_id" -# output: "$id.$key.output.h5mu" -make_observation_keys_unique: false -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/.config.vsh.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/.config.vsh.yaml deleted file mode 100644 index 57543af..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/.config.vsh.yaml +++ /dev/null @@ -1,320 +0,0 @@ -name: "grep_annotation_column" -namespace: "metadata" -version: "disable-scrublet_build" -authors: -- name: "Dries Schaumont" - roles: - - "maintainer" - info: - role: "Core Team Member" - links: - email: "dries@data-intuitive.com" - github: "DriesSchaumont" - orcid: "0000-0002-4389-0440" - linkedin: "dries-schaumont" - organizations: - - name: "Data Intuitive" - href: "https://www.data-intuitive.com" - role: "Data Scientist" -argument_groups: -- name: "Inputs" - description: "Arguments related to the input dataset." - arguments: - - type: "file" - name: "--input" - alternatives: - - "-i" - description: "Path to the input .h5mu." - info: null - example: - - "sample_path" - must_exist: true - create_parent: true - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--input_column" - description: "Column to query. If not specified, use .var_names or .obs_names,\ - \ depending on the value of --matrix" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--input_layer" - description: "Input data to use when calculating fraction of observations that\ - \ match with the query. \nOnly used when --output_fraction_column is provided.\ - \ If not specified, .X is used.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--modality" - description: "Which modality to get the annotation matrix from.\n" - info: null - example: - - "rna" - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--matrix" - description: "Matrix to fetch the column from that will be searched." - info: null - example: - - "var" - required: false - choices: - - "var" - - "obs" - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Outputs" - description: "Arguments related to how the output will be written." - arguments: - - type: "file" - name: "--output" - alternatives: - - "-o" - description: "Location of the output MuData file.\n" - info: null - example: - - "output.h5mu" - must_exist: true - create_parent: true - required: false - direction: "output" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_match_column" - description: "Name of the column to write the result to." - info: null - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_fraction_column" - description: "For the opposite axis, name of the column to write the fraction\ - \ of \nobservations that matches to the pattern.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_compression" - description: "Compression format to use for the output AnnData and/or Mudata objects.\n\ - By default no compression is applied.\n" - info: null - example: - - "gzip" - required: false - choices: - - "gzip" - - "lzf" - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Query options" - description: "Options related to the query" - arguments: - - type: "string" - name: "--regex_pattern" - description: "Regex to use to match with the input column." - info: null - example: - - "^[mM][tT]-" - required: true - direction: "input" - multiple: false - multiple_sep: ";" -resources: -- type: "python_script" - path: "script.py" - is_executable: true -- type: "file" - path: "setup_logger.py" -- type: "file" - path: "compress_h5mu.py" -- type: "file" - path: "nextflow_labels.config" - dest: "nextflow_labels.config" -description: "Perform a regex lookup on a column from the annotation matrices .obs\ - \ or .var.\nThe annotation matrix can originate from either a modality, or all modalities\ - \ (global .var or .obs).\n" -test_resources: -- type: "python_script" - path: "test.py" - is_executable: true -- type: "file" - path: "e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset_unique_obs.h5mu" -info: null -status: "enabled" -scope: - image: "public" - target: "public" -license: "MIT" -links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" -runners: -- type: "executable" - id: "executable" - docker_setup_strategy: "ifneedbepullelsecachedbuild" -- type: "nextflow" - id: "nextflow" - directives: - label: - - "singlecpu" - - "lowmem" - tag: "$id" - auto: - simplifyInput: true - simplifyOutput: false - transcript: false - publish: false - config: - labels: - mem1gb: "memory = 1000000000.B" - mem2gb: "memory = 2000000000.B" - mem5gb: "memory = 5000000000.B" - mem10gb: "memory = 10000000000.B" - mem20gb: "memory = 20000000000.B" - mem50gb: "memory = 50000000000.B" - mem100gb: "memory = 100000000000.B" - mem200gb: "memory = 200000000000.B" - mem500gb: "memory = 500000000000.B" - mem1tb: "memory = 1000000000000.B" - mem2tb: "memory = 2000000000000.B" - mem5tb: "memory = 5000000000000.B" - mem10tb: "memory = 10000000000000.B" - mem20tb: "memory = 20000000000000.B" - mem50tb: "memory = 50000000000000.B" - mem100tb: "memory = 100000000000000.B" - mem200tb: "memory = 200000000000000.B" - mem500tb: "memory = 500000000000000.B" - mem1gib: "memory = 1073741824.B" - mem2gib: "memory = 2147483648.B" - mem4gib: "memory = 4294967296.B" - mem8gib: "memory = 8589934592.B" - mem16gib: "memory = 17179869184.B" - mem32gib: "memory = 34359738368.B" - mem64gib: "memory = 68719476736.B" - mem128gib: "memory = 137438953472.B" - mem256gib: "memory = 274877906944.B" - mem512gib: "memory = 549755813888.B" - mem1tib: "memory = 1099511627776.B" - mem2tib: "memory = 2199023255552.B" - mem4tib: "memory = 4398046511104.B" - mem8tib: "memory = 8796093022208.B" - mem16tib: "memory = 17592186044416.B" - mem32tib: "memory = 35184372088832.B" - mem64tib: "memory = 70368744177664.B" - mem128tib: "memory = 140737488355328.B" - mem256tib: "memory = 281474976710656.B" - mem512tib: "memory = 562949953421312.B" - cpu1: "cpus = 1" - cpu2: "cpus = 2" - cpu5: "cpus = 5" - cpu10: "cpus = 10" - cpu20: "cpus = 20" - cpu50: "cpus = 50" - cpu100: "cpus = 100" - cpu200: "cpus = 200" - cpu500: "cpus = 500" - cpu1000: "cpus = 1000" - script: - - "includeConfig(\"nextflow_labels.config\")" - debug: false - container: "docker" -engines: -- type: "docker" - id: "docker" - image: "python:3.11-slim" - target_tag: "disable-scrublet_build" - namespace_separator: "/" - setup: - - type: "apt" - packages: - - "procps" - interactive: false - - type: "python" - user: false - packages: - - "anndata~=0.11.1" - - "mudata~=0.3.1" - script: - - "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\ - nelse: exit(1)\")" - upgrade: true - test_setup: - - type: "apt" - packages: - - "git" - interactive: false - - type: "python" - user: false - packages: - - "viashpy==0.8.0" - github: - - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" - upgrade: true - entrypoint: [] - cmd: null -build_info: - config: "src/metadata/grep_annotation_column/config.vsh.yaml" - runner: "nextflow" - engine: "docker" - output: "target/nextflow/metadata/grep_annotation_column" - executable: "target/nextflow/metadata/grep_annotation_column/main.nf" - viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" - git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" -package_config: - name: "openpipeline" - summary: "Best-practice workflows for single-cell multi-omics analyses.\n" - description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ - \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ - \nIn terms of workflows, the following has been made available, but keep in mind\ - \ that\nindividual tools and functionality can be executed as standalone components\ - \ as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n\ - \ * Ingestion: Read mapping and generating a count matrix.\n * Single sample\ - \ processing: cell filtering and doublet detection.\n * Multisample processing:\ - \ Count transformation, normalization, QC metric calulations.\n * Integration:\ - \ Clustering, integration and batch correction using single and multimodal methods.\n\ - \ * Downstream analysis workflows\n" - info: - test_resources: - - type: "s3" - path: "s3://openpipelines-data" - dest: "resources_test" - viash_version: "0.9.4" - source: "src" - target: "target" - config_mods: - - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ - .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ - )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" - keywords: - - "single-cell" - - "multimodal" - license: "MIT" - organization: "openpipelines-bio" - links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" - homepage: "https://openpipelines.bio" - documentation: "https://openpipelines.bio/fundamentals" - issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/main.nf b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/main.nf deleted file mode 100644 index c78d691..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/main.nf +++ /dev/null @@ -1,4123 +0,0 @@ -// grep_annotation_column disable-scrublet_build -// -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative -// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data -// Intuitive. -// -// The component may contain files which fall under a different license. The -// authors of this component should specify the license in the header of such -// files, or include a separate license file detailing the licenses of all included -// files. -// -// Component authors: -// * Dries Schaumont (maintainer) - -//////////////////////////// -// VDSL3 helper functions // -//////////////////////////// - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' -class UnexpectedArgumentTypeException extends Exception { - String errorIdentifier - String stage - String plainName - String expectedClass - String foundClass - - // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} - UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { - super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + - "Expected type: ${expectedClass}. Found type: ${foundClass}") - this.errorIdentifier = errorIdentifier - this.stage = stage - this.plainName = plainName - this.expectedClass = expectedClass - this.foundClass = foundClass - } -} - -/** - * Checks if the given value is of the expected type. If not, an exception is thrown. - * - * @param stage The stage of the argument (input or output) - * @param par The parameter definition - * @param value The value to check - * @param errorIdentifier The identifier to use in the error message - * @return The value, if it is of the expected type - * @throws UnexpectedArgumentTypeException If the value is not of the expected type -*/ -def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { - // expectedClass will only be != null if value is not of the expected type - def expectedClass = null - def foundClass = null - - // todo: split if need be - - if (!par.required && value == null) { - expectedClass = null - } else if (par.multiple) { - if (value !instanceof Collection) { - value = [value] - } - - // split strings - value = value.collectMany{ val -> - if (val instanceof String) { - // collect() to ensure that the result is a List and not simply an array - val.split(par.multiple_sep).collect() - } else { - [val] - } - } - - // process globs - if (par.type == "file" && par.direction == "input") { - value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() - } - - // check types of elements in list - try { - value = value.collect { listVal -> - _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) - } - } catch (UnexpectedArgumentTypeException e) { - expectedClass = "List[${e.expectedClass}]" - foundClass = "List[${e.foundClass}]" - } - } else if (par.type == "string") { - // cast to string if need be. only cast if the value is a GString - if (value instanceof GString) { - value = value as String - } - expectedClass = value instanceof String ? null : "String" - } else if (par.type == "integer") { - // cast to integer if need be - if (value !instanceof Integer) { - try { - value = value as Integer - } catch (NumberFormatException e) { - expectedClass = "Integer" - } - } - } else if (par.type == "long") { - // cast to long if need be - if (value !instanceof Long) { - try { - value = value as Long - } catch (NumberFormatException e) { - expectedClass = "Long" - } - } - } else if (par.type == "double") { - // cast to double if need be - if (value !instanceof Double) { - try { - value = value as Double - } catch (NumberFormatException e) { - expectedClass = "Double" - } - } - } else if (par.type == "float") { - // cast to float if need be - if (value !instanceof Float) { - try { - value = value as Float - } catch (NumberFormatException e) { - expectedClass = "Float" - } - } - } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { - // cast to boolean if need be - if (value !instanceof Boolean) { - try { - value = value as Boolean - } catch (Exception e) { - expectedClass = "Boolean" - } - } - } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { - // cast to path if need be - if (value instanceof String) { - value = file(value, hidden: true) - } - if (value instanceof File) { - value = value.toPath() - } - expectedClass = value instanceof Path ? null : "Path" - } else if (par.type == "file" && stage == "input" && par.direction == "output") { - // cast to string if need be - if (value !instanceof String) { - try { - value = value as String - } catch (Exception e) { - expectedClass = "String" - } - } - } else { - // didn't find a match for par.type - expectedClass = par.type - } - - if (expectedClass != null) { - if (foundClass == null) { - foundClass = value.getClass().getName() - } - throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) - } - - return value -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' -Map _processInputValues(Map inputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.required && arg.direction == "input") { - assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" - } - } - - inputs = inputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" - - value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return inputs -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - outputs = outputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && it.direction == "output" } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" - - value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return outputs -} - -void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' -class IDChecker { - final def items = [] as Set - - @groovy.transform.WithWriteLock - boolean observe(String item) { - if (items.contains(item)) { - return false - } else { - items << item - return true - } - } - - @groovy.transform.WithReadLock - boolean contains(String item) { - return items.contains(item) - } - - @groovy.transform.WithReadLock - Set getItems() { - return items.clone() - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' - -/** - * Check if the ids are unique across parameter sets - * - * @param parameterSets a list of parameter sets. - */ -private void _checkUniqueIds(List>> parameterSets) { - def ppIds = parameterSets.collect{it[0]} - assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' - -// helper functions for reading params from file // -def _getChild(parent, child) { - if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { - child - } else { - def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() - parentAbsolute.replaceAll('/[^/]*$', "/") + child - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' -/** - * Figure out the param list format based on the file extension - * - * @param param_list A String containing the path to the parameter list file. - * - * @return A String containing the format of the parameter list file. - */ -def _paramListGuessFormat(param_list) { - if (param_list !instanceof String) { - "asis" - } else if (param_list.endsWith(".csv")) { - "csv" - } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { - "json" - } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { - "yaml" - } else { - "yaml_blob" - } -} - - -/** - * Read the param list - * - * @param param_list One of the following: - * - A String containing the path to the parameter list file (csv, json or yaml), - * - A yaml blob of a list of maps (yaml_blob), - * - Or a groovy list of maps (asis). - * @param config A Map of the Viash configuration. - * - * @return A List of Maps containing the parameters. - */ -def _parseParamList(param_list, Map config) { - // first determine format by extension - def paramListFormat = _paramListGuessFormat(param_list) - - def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? - file(param_list, hidden: true) : - null - - // get the correct parser function for the detected params_list format - def paramSets = [] - if (paramListFormat == "asis") { - paramSets = param_list - } else if (paramListFormat == "yaml_blob") { - paramSets = readYamlBlob(param_list) - } else if (paramListFormat == "yaml") { - paramSets = readYaml(paramListPath) - } else if (paramListFormat == "json") { - paramSets = readJson(paramListPath) - } else if (paramListFormat == "csv") { - paramSets = readCsv(paramListPath) - } else { - error "Format of provided --param_list not recognised.\n" + - "Found: '$paramListFormat'.\n" + - "Expected: a csv file, a json file, a yaml file,\n" + - "a yaml blob or a groovy list of maps." - } - - // data checks - assert paramSets instanceof List: "--param_list should contain a list of maps" - for (value in paramSets) { - assert value instanceof Map: "--param_list should contain a list of maps" - } - - // id is argument - def idIsArgument = config.allArguments.any{it.plainName == "id"} - - // Reformat from List to List> by adding the ID as first element of a Tuple2 - paramSets = paramSets.collect({ data -> - def id = data.id - if (!idIsArgument) { - data = data.findAll{k, v -> k != "id"} - } - [id, data] - }) - - // Split parameters with 'multiple: true' - paramSets = paramSets.collect({ id, data -> - data = _splitParams(data, config) - [id, data] - }) - - // The paths of input files inside a param_list file may have been specified relatively to the - // location of the param_list file. These paths must be made absolute. - if (paramListPath) { - paramSets = paramSets.collect({ id, data -> - def new_data = data.collectEntries{ parName, parValue -> - def par = config.allArguments.find{it.plainName == parName} - if (par && par.type == "file" && par.direction == "input") { - if (parValue instanceof Collection) { - parValue = parValue.collectMany{path -> - def x = _resolveSiblingIfNotAbsolute(path, paramListPath) - x instanceof Collection ? x : [x] - } - } else { - parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) - } - } - [parName, parValue] - } - [id, new_data] - }) - } - - return paramSets -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' -/** - * Split parameters for arguments that accept multiple values using their separator - * - * @param paramList A Map containing parameters to split. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A Map of parameters where the parameter values have been split into a list using - * their seperator. - */ -Map _splitParams(Map parValues, Map config){ - def parsedParamValues = parValues.collectEntries { parName, parValue -> - def parameterSettings = config.allArguments.find({it.plainName == parName}) - - if (!parameterSettings) { - // if argument is not found, do not alter - return [parName, parValue] - } - if (parameterSettings.multiple) { // Check if parameter can accept multiple values - if (parValue instanceof Collection) { - parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } - } else if (parValue instanceof String) { - parValue = parValue.split(parameterSettings.multiple_sep) - } else if (parValue == null) { - parValue = [] - } else { - parValue = [ parValue ] - } - parValue = parValue.flatten() - } - // For all parameters check if multiple values are only passed for - // arguments that allow it. Quietly simplify lists of length 1. - if (!parameterSettings.multiple && parValue instanceof Collection) { - assert parValue.size() == 1 : - "Error: argument ${parName} has too many values.\n" + - " Expected amount: 1. Found: ${parValue.size()}" - parValue = parValue[0] - } - [parName, parValue] - } - return parsedParamValues -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' -/** - * Parse nextflow parameters based on settings defined in a viash config. - * Return a list of parameter sets, each parameter set corresponding to - * an event in a nextflow channel. The output from this function can be used - * with Channel.fromList to create a nextflow channel with Vdsl3 formatted - * events. - * - * This function performs: - * - A filtering of the params which can be found in the config file. - * - Process the params_list argument which allows a user to to initialise - * a Vsdl3 channel with multiple parameter sets. Possible formats are - * csv, json, yaml, or simply a yaml_blob. A csv should have column names - * which correspond to the different arguments of this pipeline. A json or a yaml - * file should be a list of maps, each of which has keys corresponding to the - * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. - * When passing a csv, json or yaml, relative path names are relativized to the - * location of the parameter file. - * - Combine the parameter sets into a vdsl3 Channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A list of parameters with the first element of the event being - * the event ID and the second element containing a map of the parsed parameters. - */ - -private List>> _paramsToParamSets(Map params, Map config){ - // todo: fetch key from run args - def key_ = config.name - - /* parse regular parameters (not in param_list) */ - /*************************************************/ - def globalParams = config.allArguments - .findAll { params.containsKey(it.plainName) } - .collectEntries { [ it.plainName, params[it.plainName] ] } - def globalID = params.get("id", null) - - /* process params_list arguments */ - /*********************************/ - def paramList = params.containsKey("param_list") && params.param_list != null ? - params.param_list : [] - // if (paramList instanceof String) { - // paramList = [paramList] - // } - // def paramSets = paramList.collectMany{ _parseParamList(it, config) } - // TODO: be able to process param_list when it is a list of strings - def paramSets = _parseParamList(paramList, config) - if (paramSets.isEmpty()) { - paramSets = [[null, [:]]] - } - - /* combine arguments into channel */ - /**********************************/ - def processedParams = paramSets.indexed().collect{ index, tup -> - // Process ID - def id = tup[0] ?: globalID - - if (workflow.stubRun && !id) { - // if stub run, explicitly add an id if missing - id = "stub${index}" - } - assert id != null: "Each parameter set should have at least an 'id'" - - // Process params - def parValues = globalParams + tup[1] - // // Remove parameters which are null, if the default is also null - // parValues = parValues.collectEntries{paramName, paramValue -> - // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) - // if ( paramValue != null || parameterSettings.get("default", null) != null ) { - // [paramName, paramValue] - // } - // } - parValues = parValues.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" - - if (par == null) { - return [:] - } - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - - [ name, value ] - } - - [id, parValues] - } - - // Check if ids (first element of each list) is unique - _checkUniqueIds(processedParams) - return processedParams -} - -/** - * Parse nextflow parameters based on settings defined in a viash config - * and return a nextflow channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A nextflow Channel with events. Events are formatted as a tuple that contains - * first contains the ID of the event and as second element holds a parameter map. - * - * - */ -def channelFromParams(Map params, Map config) { - def processedParams = _paramsToParamSets(params, config) - return Channel.fromList(processedParams) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' -def checkUniqueIds(Map args) { - def stopOnError = args.stopOnError == null ? args.stopOnError : true - - def idChecker = new IDChecker() - - return filter { tup -> - if (!idChecker.observe(tup[0])) { - if (stopOnError) { - error "Duplicate id: ${tup[0]}" - } else { - log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" - return false - } - } - return true - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' -// This helper file will be deprecated soon -preprocessInputsDeprecationWarningPrinted = false - -def preprocessInputsDeprecationWarning() { - if (!preprocessInputsDeprecationWarningPrinted) { - preprocessInputsDeprecationWarningPrinted = true - System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") - } -} - -/** - * Generate a nextflow Workflow that allows processing a channel of - * Vdsl3 formatted events and apply a Viash config to them: - * - Gather default parameters from the Viash config and make - * sure that they are correctly formatted (see applyConfig method). - * - Format the input parameters (also using the applyConfig method). - * - Apply the default parameter to the input parameters. - * - Do some assertions: - * ~ Check if the event IDs in the channel are unique. - * - * The events in the channel are formatted as tuples, with the - * first element of the tuples being a unique id of the parameter set, - * and the second element containg the the parameters themselves. - * Optional extra elements of the tuples will be passed to the output as is. - * - * @param args A map that must contain a 'config' key that points - * to a parsed config (see readConfig()). Optionally, a - * 'key' key can be provided which can be used to create a unique - * name for the workflow process. - * - * @return A workflow that allows processing a channel of Vdsl3 formatted events - * and apply a Viash config to them. - */ -def preprocessInputs(Map args) { - preprocessInputsDeprecationWarning() - - def config = args.config - assert config instanceof Map : - "Error in preprocessInputs: config must be a map. " + - "Expected class: Map. Found: config.getClass() is ${config.getClass()}" - def key_ = args.key ?: config.name - - // Get different parameter types (used throughout this function) - def defaultArgs = config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - map { tup -> - def id = tup[0] - def data = tup[1] - def passthrough = tup.drop(2) - - def new_data = (defaultArgs + data).collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - - if (par != null) { - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - } - - [ name, value ] - } - - [ id, new_data ] + passthrough - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component config. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component config. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component config. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component config. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runComponents(Map args) { - log.warn("runComponents is deprecated, use runEach instead") - assert args.components: "runComponents should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runComponents" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def id_ = args.id - - workflow runComponentsWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def comp_config = comp_.config - - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_config) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - // def new_id = id_(tup[0], tup[1], comp_config) - def new_id = tup[0] - if (id_ instanceof String) { - new_id = id_ - } else if (id_ instanceof Closure) { - new_id = id_(new_id, tup[1], comp_config) - } - [new_id] + tup.drop(1) - } - : filter_ch - def data_ch = id_ch | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_config) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_config) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - post_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runComponentsWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component itself. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component itself. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component itself. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component itself. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runEach(Map args) { - assert args.components: "runEach should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runEach" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def runIf_ = args.runIf - def id_ = args.id - - assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." - - workflow runEachWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - def new_id = id_ - if (new_id instanceof Closure) { - new_id = new_id(tup[0], tup[1], comp_) - } - assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" - [new_id] + tup.drop(1) - } - : filter_ch - def chPassthrough = null - def chRun = null - if (runIf_) { - def idRunIfBranch = id_ch.branch{ tup -> - run: runIf_(tup[0], tup[1], comp_) - passthrough: true - } - chPassthrough = idRunIfBranch.passthrough - chRun = idRunIfBranch.run - } else { - chRun = id_ch - chPassthrough = Channel.empty() - } - def data_ch = chRun | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - def return_ch = post_ch - | concat(chPassthrough) - - return_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runEachWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' -/** - * Join sourceChannel to targetChannel - * - * This function joins the sourceChannel to the targetChannel. - * However, each id in the targetChannel must be present in the - * sourceChannel. If _meta.join_id exists in the targetChannel, that is - * used as an id instead. If the id doesn't match any id in the sourceChannel, - * an error is thrown. - */ - -def safeJoin(targetChannel, sourceChannel, key) { - def sourceIDs = new IDChecker() - - def sourceCheck = sourceChannel - | map { tup -> - sourceIDs.observe(tup[0]) - tup - } - def targetCheck = targetChannel - | map { tup -> - def id = tup[0] - - if (!sourceIDs.contains(id)) { - error ( - "Error in module '${key}' when merging output with original state.\n" + - " Reason: output with id '${id}' could not be joined with source channel.\n" + - " If the IDs in the output channel differ from the input channel,\n" + - " please set `tup[1]._meta.join_id to the original ID.\n" + - " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + - " Unexpected ID in the output channel: '${id}'.\n" + - " Example input event: [\"id\", [input: file(...)]],\n" + - " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" - ) - } - // TODO: add link to our documentation on how to fix this - - tup - } - - sourceCheck.cross(targetChannel) - | map{ left, right -> - right + left.drop(1) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' -def _processArgument(arg) { - arg.multiple = arg.multiple != null ? arg.multiple : false - arg.required = arg.required != null ? arg.required : false - arg.direction = arg.direction != null ? arg.direction : "input" - arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" - arg.plainName = arg.name.replaceAll("^-*", "") - - if (arg.type == "file") { - arg.must_exist = arg.must_exist != null ? arg.must_exist : true - arg.create_parent = arg.create_parent != null ? arg.create_parent : true - } - - // add default values to output files which haven't already got a default - if (arg.type == "file" && arg.direction == "output" && arg.default == null) { - def mult = arg.multiple ? "_*" : "" - def extSearch = "" - if (arg.default != null) { - extSearch = arg.default - } else if (arg.example != null) { - extSearch = arg.example - } - if (extSearch instanceof List) { - extSearch = extSearch[0] - } - def extSearchResult = extSearch.find("\\.[^\\.]+\$") - def ext = extSearchResult != null ? extSearchResult : "" - arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" - if (arg.multiple) { - arg.default = [arg.default] - } - } - - if (!arg.multiple) { - if (arg.default != null && arg.default instanceof List) { - arg.default = arg.default[0] - } - if (arg.example != null && arg.example instanceof List) { - arg.example = arg.example[0] - } - } - - if (arg.type == "boolean_true") { - arg.default = false - } - if (arg.type == "boolean_false") { - arg.default = true - } - - arg -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' -def addGlobalArguments(config) { - def localConfig = [ - "argument_groups": [ - [ - "name": "Nextflow input-output arguments", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "arguments" : [ - [ - 'name': '--publish_dir', - 'required': true, - 'type': 'string', - 'description': 'Path to an output directory.', - 'example': 'output/', - 'multiple': false - ], - [ - 'name': '--param_list', - 'required': false, - 'type': 'string', - 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. - | - |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. - |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. - |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. - |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. - | - |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), - 'example': 'my_params.yaml', - 'multiple': false, - 'hidden': true - ] - // TODO: allow multiple: true in param_list? - // TODO: allow to specify a --param_list_regex to filter the param_list? - // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? - ] - ] - ] - ] - - return processConfig(_mergeMap(config, localConfig)) -} - -def _mergeMap(Map lhs, Map rhs) { - return rhs.inject(lhs.clone()) { map, entry -> - if (map[entry.key] instanceof Map && entry.value instanceof Map) { - map[entry.key] = _mergeMap(map[entry.key], entry.value) - } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { - map[entry.key] += entry.value - } else { - map[entry.key] = entry.value - } - return map - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' -def _generateArgumentHelp(param) { - // alternatives are not supported - // def names = param.alternatives ::: List(param.name) - - def unnamedProps = [ - ["required parameter", param.required], - ["multiple values allowed", param.multiple], - ["output", param.direction.toLowerCase() == "output"], - ["file must exist", param.type == "file" && param.must_exist] - ].findAll{it[1]}.collect{it[0]} - - def dflt = null - if (param.default != null) { - if (param.default instanceof List) { - dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - dflt = param.default.toString() - } - } - def example = null - if (param.example != null) { - if (param.example instanceof List) { - example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - example = param.example.toString() - } - } - def min = param.min?.toString() - def max = param.max?.toString() - - def escapeChoice = { choice -> - def s1 = choice.replaceAll("\\n", "\\\\n") - def s2 = s1.replaceAll("\"", """\\\"""") - s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 - } - def choices = param.choices == null ? - null : - "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" - - def namedPropsStr = [ - ["type", ([param.type] + unnamedProps).join(", ")], - ["default", dflt], - ["example", example], - ["choices", choices], - ["min", min], - ["max", max] - ] - .findAll{it[1]} - .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} - .join("") - - def descStr = param.description == null ? - "" : - _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") - - "\n --" + param.plainName + - namedPropsStr + - descStr -} - -// Based on Helper.generateHelp() in Helper.scala -def _generateHelp(config) { - def fun = config - - // PART 1: NAME AND VERSION - def nameStr = fun.name + - (fun.version == null ? "" : " " + fun.version) - - // PART 2: DESCRIPTION - def descrStr = fun.description == null ? - "" : - "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") - - // PART 3: Usage - def usageStr = fun.usage == null ? - "" : - "\n\nUsage:\n" + fun.usage.trim() - - // PART 4: Options - def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> - def name = argGroup.name - def descriptionStr = argGroup.description == null ? - "" : - "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" - def arguments = argGroup.arguments.collect{arg -> - arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg - }.findAll{it != null} - def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} - - "\n\n$name:" + - descriptionStr + - argumentStrs.join("\n") - } - - // FINAL: combine - def out = nameStr + - descrStr + - usageStr + - argGroupStrs.join("") - - return out -} - -// based on Format._paragraphWrap -def _paragraphWrap(str, maxLength) { - def outLines = [] - str.split("\n").each{par -> - def words = par.split("\\s").toList() - - def word = null - def line = words.pop() - while(!words.isEmpty()) { - word = words.pop() - if (line.length() + word.length() + 1 <= maxLength) { - line = line + " " + word - } else { - outLines.add(line) - line = word - } - } - if (words.isEmpty()) { - outLines.add(line) - } - } - return outLines -} - -def helpMessage(config) { - if (params.containsKey("help") && params.help) { - def mergedConfig = addGlobalArguments(config) - def helpStr = _generateHelp(mergedConfig) - println(helpStr) - exit 0 - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' -def processConfig(config) { - // set defaults for arguments - config.arguments = - (config.arguments ?: []).collect{_processArgument(it)} - - // set defaults for argument_group arguments - config.argument_groups = - (config.argument_groups ?: []).collect{grp -> - grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} - grp - } - - // create combined arguments list - config.allArguments = - config.arguments + - config.argument_groups.collectMany{it.arguments} - - // add missing argument groups (based on Functionality::allArgumentGroups()) - def argGroups = config.argument_groups - if (argGroups.any{it.name.toLowerCase() == "arguments"}) { - argGroups = argGroups.collect{ grp -> - if (grp.name.toLowerCase() == "arguments") { - grp = grp + [ - arguments: grp.arguments + config.arguments - ] - } - grp - } - } else { - argGroups = argGroups + [ - name: "Arguments", - arguments: config.arguments - ] - } - config.allArgumentGroups = argGroups - - config -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' - -def readConfig(file) { - def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) - processConfig(config) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' -/** - * Resolve a path relative to the current file. - * - * @param str The path to resolve, as a String. - * @param parentPath The path to resolve relative to, as a Path. - * - * @return The path that may have been resovled, as a Path. - */ -def _resolveSiblingIfNotAbsolute(str, parentPath) { - if (str !instanceof String) { - return str - } - if (!_stringIsAbsolutePath(str)) { - return parentPath.resolveSibling(str) - } else { - return file(str, hidden: true) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' -/** - * Check whether a path as a string is absolute. - * - * In the past, we tried using `file(., relative: true).isAbsolute()`, - * but the 'relative' option was added in 22.10.0. - * - * @param path The path to check, as a String. - * - * @return Whether the path is absolute, as a boolean. - */ -def _stringIsAbsolutePath(path) { - def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ - - assert path instanceof String - return _resolve_URL_PROTOCOL.matcher(path).matches() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' -class CustomTraceObserver implements nextflow.trace.TraceObserver { - List traces - - CustomTraceObserver(List traces) { - this.traces = traces - } - - @Override - void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } - - @Override - void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } -} - -def collectTraces() { - def traces = Collections.synchronizedList([]) - - // add custom trace observer which stores traces in the traces object - session.observers.add(new CustomTraceObserver(traces)) - - traces -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' -/** - * Performs a deep clone of the given object. - * @param x an object - */ -def deepClone(x) { - iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' -def getPublishDir() { - return params.containsKey("publish_dir") ? params.publish_dir : - params.containsKey("publishDir") ? params.publishDir : - null -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' - -// Recurse upwards until we find a '.build.yaml' file -def _findBuildYamlFile(pathPossiblySymlink) { - def path = pathPossiblySymlink.toRealPath() - def child = path.resolve(".build.yaml") - if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { - return child - } else { - def parent = path.getParent() - if (parent == null) { - return null - } else { - return _findBuildYamlFile(parent) - } - } -} - -// get the root of the target folder -def getRootDir() { - def dir = _findBuildYamlFile(meta.resources_dir) - assert dir != null: "Could not find .build.yaml in the folder structure" - dir.getParent() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' -/** - * Recursively apply a function over the leaves of an object. - * @param obj The object to iterate over. - * @param fun The function to apply to each value. - * @return The object with the function applied to each value. - */ -def iterateMap(obj, fun) { - if (obj instanceof List && obj !instanceof String) { - return obj.collect{item -> - iterateMap(item, fun) - } - } else if (obj instanceof Map) { - return obj.collectEntries{key, item -> - [key.toString(), iterateMap(item, fun)] - } - } else { - return fun(obj) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' -/** - * A view for printing the event of each channel as a YAML blob. - * This is useful for debugging. - */ -def niceView() { - workflow niceViewWf { - take: input - main: - output = input - | view{toYamlBlob(it)} - emit: output - } - return niceViewWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' - -def readCsv(file_path) { - def output = [] - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - - // todo: allow escaped quotes in string - // todo: allow single quotes? - def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') - def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - - def br = java.nio.file.Files.newBufferedReader(inputFile) - - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field - } - } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) - } - } - - output -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' -def readJson(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parse(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' -def readJsonBlob(str) { - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parseText(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' -// Custom constructor to modify how certain objects are parsed from YAML -class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { - Path root - - class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { - public Object construct(org.yaml.snakeyaml.nodes.Node node) { - String filename = (String) constructScalar(node); - if (root != null) { - return root.resolve(filename); - } - return java.nio.file.Paths.get(filename); - } - } - - CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { - super(options) - this.root = root - // Handling !file tag and parse it back to a File type - this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) - } -} - -def readTaggedYaml(Path path) { - def options = new org.yaml.snakeyaml.LoaderOptions() - def constructor = new CustomConstructor(options, path.getParent()) - def yaml = new org.yaml.snakeyaml.Yaml(constructor) - return yaml.load(path.text) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' -def readYaml(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' -def readYamlBlob(str) { - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' -String toJsonBlob(data) { - return groovy.json.JsonOutput.toJson(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' -// Custom representer to modify how certain objects are represented in YAML -class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { - Path relativizer - - class RepresentPath implements org.yaml.snakeyaml.representer.Represent { - public String getFileName(Object obj) { - if (obj instanceof File) { - obj = ((File) obj).toPath(); - } - if (obj !instanceof Path) { - throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); - } - def path = (Path) obj; - - if (relativizer != null) { - return relativizer.relativize(path).toString() - } else { - return path.toString() - } - } - - public org.yaml.snakeyaml.nodes.Node representData(Object data) { - String filename = getFileName(data); - def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); - return representScalar(tag, filename); - } - } - CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { - super(options) - this.relativizer = relativizer - this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) - this.representers.put(Path, new RepresentPath()) - this.representers.put(File, new RepresentPath()) - } -} - -String toTaggedYamlBlob(data) { - return toRelativeTaggedYamlBlob(data, null) -} -String toRelativeTaggedYamlBlob(data, Path relativizer) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - def representer = new CustomRepresenter(options, relativizer) - def yaml = new org.yaml.snakeyaml.Yaml(representer, options) - return yaml.dump(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' -String toYamlBlob(data) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - options.setPrettyFlow(true) - def yaml = new org.yaml.snakeyaml.Yaml(options) - def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) - return yaml.dump(cleanData) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' -void writeJson(data, file) { - assert data: "writeJson: data should not be null" - assert file: "writeJson: file should not be null" - file.write(toJsonBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' -void writeYaml(data, file) { - assert data: "writeYaml: data should not be null" - assert file: "writeYaml: file should not be null" - file.write(toYamlBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' -def findStates(Map params, Map config) { - def auto_config = deepClone(config) - def auto_params = deepClone(params) - - auto_config = auto_config.clone() - // override arguments - auto_config.argument_groups = [] - auto_config.arguments = [ - [ - type: "string", - name: "--id", - description: "A dummy identifier", - required: false - ], - [ - type: "file", - name: "--input_states", - example: "/path/to/input/directory/**/state.yaml", - description: "Path to input directory containing the datasets to be integrated.", - required: true, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--filter", - example: "foo/.*/state.yaml", - description: "Regex to filter state files by path.", - required: false - ], - // to do: make this a yaml blob? - [ - type: "string", - name: "--rename_keys", - example: ["newKey1:oldKey1", "newKey2:oldKey2"], - description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", - required: false, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--settings", - example: '{"output_dataset": "dataset.h5ad", "k": 10}', - description: "Global arguments as a JSON glob to be passed to all components.", - required: false - ] - ] - if (!(auto_params.containsKey("id"))) { - auto_params["id"] = "auto" - } - - // run auto config through processConfig once more - auto_config = processConfig(auto_config) - - workflow findStatesWf { - helpMessage(auto_config) - - output_ch = - channelFromParams(auto_params, auto_config) - | flatMap { autoId, args -> - - def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] - - // look for state files in input dir - def stateFiles = args.input_states - - // filter state files by regex - if (args.filter) { - stateFiles = stateFiles.findAll{ stateFile -> - def stateFileStr = stateFile.toString() - def matcher = stateFileStr =~ args.filter - matcher.matches()} - } - - // read in states - def states = stateFiles.collect { stateFile -> - def state_ = readTaggedYaml(stateFile) - [state_.id, state_] - } - - // construct renameMap - if (args.rename_keys) { - def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(":") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" - split - } - - // rename keys in state, only let states through which have all keys - // also add global settings - states = states.collectMany{id, state -> - def newState = [:] - - for (key in renameMap.keySet()) { - def origKey = renameMap[key] - if (!(state.containsKey(origKey))) { - return [] - } - newState[key] = state[origKey] - } - - [[id, globalSettings + newState]] - } - } - - states - } - emit: - output_ch - } - - return findStatesWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' -def joinStates(Closure apply_) { - workflow joinStatesWf { - take: input_ch - main: - output_ch = input_ch - | toSortedList - | filter{ it.size() > 0 } - | map{ tups -> - def ids = tups.collect{it[0]} - def states = tups.collect{it[1]} - apply_(ids, states) - } - - emit: output_ch - } - return joinStatesWf -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' -def publishFiles(Map args) { - def key_ = args.get("key") - - assert key_ != null : "publishFiles: key must be specified" - - workflow publishFilesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] - - [id_, inputFiles_, outputFilenames_] - } - | publishFilesProc - emit: input_ch - } - return publishFilesWf -} - -process publishFilesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) - output: - tuple val(id), path{outputFiles} - script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } - """ - echo "Copying output files to destination folder" - ${copyCommands.join("\n ")} - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishFilesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishFilesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishFilesByConfig: key must be specified" - - workflow publishFilesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output OR multiple channels were emitted - // and the output was just not added to using the channel - // that is now being parsed - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[inputPath: [], outputFilename: []]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def inputPath = val instanceof File ? val.toPath() : val - [inputPath: inputPath, outputFilename: filename_ix] - } - def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] - } else { - def value_ = java.nio.file.Paths.get(filename) - def inputPath = value instanceof File ? value.toPath() : value - return [[inputPath: [inputPath], outputFilename: [filename]]] - } - } - - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} - - - [id_, inputPaths, outputFilenames] - } - | publishFilesProc - emit: input_ch - } - return publishFilesSimpleWf -} - - - - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' -def collectFiles(obj) { - if (obj instanceof java.io.File || obj instanceof Path) { - return [obj] - } else if (obj instanceof List && obj !instanceof String) { - return obj.collectMany{item -> - collectFiles(item) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectFiles(item) - } - } else { - return [] - } -} - -/** - * Recurse through a state and collect all input files and their target output filenames. - * @param obj The state to recurse through. - * @param prefix The prefix to prepend to the output filenames. - */ -def collectInputOutputPaths(obj, prefix) { - if (obj instanceof File || obj instanceof Path) { - def path = obj instanceof Path ? obj : obj.toPath() - def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" - def newFilename = prefix + ext - return [[obj, newFilename]] - } else if (obj instanceof List && obj !instanceof String) { - return obj.withIndex().collectMany{item, ix -> - collectInputOutputPaths(item, prefix + "_" + ix) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectInputOutputPaths(item, prefix + "." + key) - } - } else { - return [] - } -} - -def publishStates(Map args) { - def key_ = args.get("key") - def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) - - assert key_ != null : "publishStates: key must be specified" - - workflow publishStatesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - - def yamlFilename = yamlTemplate_ - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - - // TODO: do the pathnames in state_ match up with the outputFilenames_? - - // convert state to yaml blob - def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesWf -} -process publishStatesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), val(yamlBlob), val(yamlFile) - output: - tuple val(id), path{[yamlFile]} - script: - """ - mkdir -p "\$(dirname '${yamlFile}')" - echo "Storing state as yaml" - cat > '${yamlFile}' << HERE -${yamlBlob} -HERE - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishStatesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishStatesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishStatesByConfig: key must be specified" - - workflow publishStatesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - // TODO: allow overriding the state.yaml template - // TODO TODO: if auto.publish == "state", add output_state as an argument - def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' - def yamlFilename = yamlTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - - // the processed state is a list of [key, value] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - (key, value) are the tuples that will be saved to the state.yaml file - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[key: plainName_, value: value]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def value_ = java.nio.file.Paths.get(filename_ix) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - return value_ - } - return [["key": plainName_, "value": outputPerFile]] - } else { - def value_ = java.nio.file.Paths.get(filename) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - def inputPath = value instanceof File ? value.toPath() : value - return [["key": plainName_, value: value_]] - } - } - - - def updatedState_ = processedState.collectEntries{[it.key, it.value]} - - // convert state to yaml blob - def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesSimpleWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' -def setState(fun) { - assert fun instanceof Closure || fun instanceof Map || fun instanceof List : - "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" - - // if fun is a List, convert to map - if (fun instanceof List) { - // check whether fun is a list[string] - assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" - fun = fun.collectEntries{[it, it]} - } - - // if fun is a map, convert to closure - if (fun instanceof Map) { - // check whether fun is a map[string, string] - assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" - assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" - def funMap = fun.clone() - // turn the map into a closure to be used later on - fun = { id_, state_ -> - assert state_ instanceof Map : "Error in setState: the state is not a Map" - funMap.collectMany{newkey, origkey -> - if (state_.containsKey(origkey)) { - [[newkey, state_[origkey]]] - } else { - [] - } - }.collectEntries() - } - } - - map { tup -> - def id = tup[0] - def state = tup[1] - def unfilteredState = fun(id, state) - def newState = unfilteredState.findAll{key, val -> val != null} - [id, newState] + tup.drop(2) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' -// TODO: unit test processAuto -def processAuto(Map auto) { - // remove null values - auto = auto.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] - def unexpectedKeys = auto.keySet() - expectedKeys - assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" - - // check auto.simplifyInput - assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" - - // check auto.simplifyOutput - assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" - - // check auto.transcript - assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" - - // check auto.publish - assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" - - return auto.subMap(expectedKeys) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' -def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { - assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" - map.forEach { key, val -> - assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" - } - requiredKeys.forEach { requiredKey -> - assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" - } -} - -// TODO: unit test processDirectives -def processDirectives(Map drctv) { - // remove null values - drctv = drctv.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = [ - "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" - ] - def unexpectedKeys = drctv.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" - - /* DIRECTIVE accelerator - accepted examples: - - [ limit: 4, type: "nvidia-tesla-k80" ] - */ - if (drctv.containsKey("accelerator")) { - assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") - } - - /* DIRECTIVE afterScript - accepted examples: - - "source /cluster/bin/cleanup" - */ - if (drctv.containsKey("afterScript")) { - assert drctv["afterScript"] instanceof CharSequence - } - - /* DIRECTIVE beforeScript - accepted examples: - - "source /cluster/bin/setup" - */ - if (drctv.containsKey("beforeScript")) { - assert drctv["beforeScript"] instanceof CharSequence - } - - /* DIRECTIVE cache - accepted examples: - - true - - false - - "deep" - - "lenient" - */ - if (drctv.containsKey("cache")) { - assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean - if (drctv["cache"] instanceof CharSequence) { - assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" - } - } - - /* DIRECTIVE conda - accepted examples: - - "bwa=0.7.15" - - "bwa=0.7.15 fastqc=0.11.5" - - ["bwa=0.7.15", "fastqc=0.11.5"] - */ - if (drctv.containsKey("conda")) { - if (drctv["conda"] instanceof List) { - drctv["conda"] = drctv["conda"].join(" ") - } - assert drctv["conda"] instanceof CharSequence - } - - /* DIRECTIVE container - accepted examples: - - "foo/bar:tag" - - [ registry: "reg", image: "im", tag: "ta" ] - is transformed to "reg/im:ta" - - [ image: "im" ] - is transformed to "im:latest" - */ - if (drctv.containsKey("container")) { - assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence - if (drctv["container"] instanceof Map) { - def m = drctv["container"] - assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") - def part1 = - System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : - params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? - m.registry ? m.registry + "/" : - "" - def part2 = m.image - def part3 = m.tag ? ":" + m.tag : ":latest" - drctv["container"] = part1 + part2 + part3 - } - } - - /* DIRECTIVE containerOptions - accepted examples: - - "--foo bar" - - ["--foo bar", "-f b"] - */ - if (drctv.containsKey("containerOptions")) { - if (drctv["containerOptions"] instanceof List) { - drctv["containerOptions"] = drctv["containerOptions"].join(" ") - } - assert drctv["containerOptions"] instanceof CharSequence - } - - /* DIRECTIVE cpus - accepted examples: - - 1 - - 10 - */ - if (drctv.containsKey("cpus")) { - assert drctv["cpus"] instanceof Integer - } - - /* DIRECTIVE disk - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("disk")) { - assert drctv["disk"] instanceof CharSequence - // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE echo - accepted examples: - - true - - false - */ - if (drctv.containsKey("echo")) { - assert drctv["echo"] instanceof Boolean - } - - /* DIRECTIVE errorStrategy - accepted examples: - - "terminate" - - "finish" - */ - if (drctv.containsKey("errorStrategy")) { - assert drctv["errorStrategy"] instanceof CharSequence - assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" - } - - /* DIRECTIVE executor - accepted examples: - - "local" - - "sge" - */ - if (drctv.containsKey("executor")) { - assert drctv["executor"] instanceof CharSequence - assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" - } - - /* DIRECTIVE machineType - accepted examples: - - "n1-highmem-8" - */ - if (drctv.containsKey("machineType")) { - assert drctv["machineType"] instanceof CharSequence - } - - /* DIRECTIVE maxErrors - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxErrors")) { - assert drctv["maxErrors"] instanceof Integer - } - - /* DIRECTIVE maxForks - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxForks")) { - assert drctv["maxForks"] instanceof Integer - } - - /* DIRECTIVE maxRetries - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxRetries")) { - assert drctv["maxRetries"] instanceof Integer - } - - /* DIRECTIVE memory - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("memory")) { - assert drctv["memory"] instanceof CharSequence - // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE module - accepted examples: - - "ncbi-blast/2.2.27" - - "ncbi-blast/2.2.27:t_coffee/10.0" - - ["ncbi-blast/2.2.27", "t_coffee/10.0"] - */ - if (drctv.containsKey("module")) { - if (drctv["module"] instanceof List) { - drctv["module"] = drctv["module"].join(":") - } - assert drctv["module"] instanceof CharSequence - } - - /* DIRECTIVE penv - accepted examples: - - "smp" - */ - if (drctv.containsKey("penv")) { - assert drctv["penv"] instanceof CharSequence - } - - /* DIRECTIVE pod - accepted examples: - - [ label: "key", value: "val" ] - - [ annotation: "key", value: "val" ] - - [ env: "key", value: "val" ] - - [ [label: "l", value: "v"], [env: "e", value: "v"]] - */ - if (drctv.containsKey("pod")) { - if (drctv["pod"] instanceof Map) { - drctv["pod"] = [ drctv["pod"] ] - } - assert drctv["pod"] instanceof List - drctv["pod"].forEach { pod -> - assert pod instanceof Map - // TODO: should more checks be added? - // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod - // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? - } - } - - /* DIRECTIVE publishDir - accepted examples: - - [] - - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] - - "/path/to/dir" - is transformed to [[ path: "/path/to/dir" ]] - - [ path: "/path/to/dir", mode: "cache" ] - is transformed to [[ path: "/path/to/dir", mode: "cache" ]] - */ - // TODO: should we also look at params["publishDir"]? - if (drctv.containsKey("publishDir")) { - def pblsh = drctv["publishDir"] - - // check different options - assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence - - // turn into list if not already so - // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. - pblsh = pblsh instanceof List ? pblsh : [ pblsh ] - - // check elements of publishDir - pblsh = pblsh.collect{ elem -> - // turn into map if not already so - elem = elem instanceof CharSequence ? [ path: elem ] : elem - - // check types and keys - assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" - assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") - - // check elements in map - assert elem.containsKey("path") - assert elem["path"] instanceof CharSequence - if (elem.containsKey("mode")) { - assert elem["mode"] instanceof CharSequence - assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] - } - if (elem.containsKey("overwrite")) { - assert elem["overwrite"] instanceof Boolean - } - if (elem.containsKey("pattern")) { - assert elem["pattern"] instanceof CharSequence - } - if (elem.containsKey("saveAs")) { - assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" - } - if (elem.containsKey("enabled")) { - assert elem["enabled"] instanceof Boolean - } - - // return final result - elem - } - // store final directive - drctv["publishDir"] = pblsh - } - - /* DIRECTIVE queue - accepted examples: - - "long" - - "short,long" - - ["short", "long"] - */ - if (drctv.containsKey("queue")) { - if (drctv["queue"] instanceof List) { - drctv["queue"] = drctv["queue"].join(",") - } - assert drctv["queue"] instanceof CharSequence - } - - /* DIRECTIVE label - accepted examples: - - "big_mem" - - "big_cpu" - - ["big_mem", "big_cpu"] - */ - if (drctv.containsKey("label")) { - if (drctv["label"] instanceof CharSequence) { - drctv["label"] = [ drctv["label"] ] - } - assert drctv["label"] instanceof List - drctv["label"].forEach { label -> - assert label instanceof CharSequence - // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") - // ^ does not allow closures - } - } - - /* DIRECTIVE scratch - accepted examples: - - true - - "/path/to/scratch" - - '$MY_PATH_TO_SCRATCH' - - "ram-disk" - */ - if (drctv.containsKey("scratch")) { - assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence - } - - /* DIRECTIVE storeDir - accepted examples: - - "/path/to/storeDir" - */ - if (drctv.containsKey("storeDir")) { - assert drctv["storeDir"] instanceof CharSequence - } - - /* DIRECTIVE stageInMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageInMode")) { - assert drctv["stageInMode"] instanceof CharSequence - assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] - } - - /* DIRECTIVE stageOutMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageOutMode")) { - assert drctv["stageOutMode"] instanceof CharSequence - assert drctv["stageOutMode"] in ["copy", "move", "rsync"] - } - - /* DIRECTIVE tag - accepted examples: - - "foo" - - '$id' - */ - if (drctv.containsKey("tag")) { - assert drctv["tag"] instanceof CharSequence - } - - /* DIRECTIVE time - accepted examples: - - "1h" - - "2days" - - "1day 6hours 3minutes 30seconds" - */ - if (drctv.containsKey("time")) { - assert drctv["time"] instanceof CharSequence - // todo: validation regex? - } - - return drctv -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' -def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { - // override defaults with args - def workflowArgs = defaultWfArgs + args - - // check whether 'key' exists - assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" - - // if 'key' is a closure, apply it to the original key - if (workflowArgs["key"] instanceof Closure) { - workflowArgs["key"] = workflowArgs["key"](meta.config.name) - } - def key = workflowArgs["key"] - assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" - assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" - - // check for any unexpected keys - def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] - def unexpectedKeys = workflowArgs.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" - assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" - workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" - assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" - workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) - - // auto define publish, if so desired - if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + - // " Example: params.publish_dir = \"./output/\"" - def publishDir = getPublishDir() - - if (publishDir != null) { - workflowArgs.directives.publishDir = [[ - path: publishDir, - saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default - mode: "copy" - ]] - } - } - - // auto define transcript, if so desired - if (workflowArgs.auto.transcript == true) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + - // " Example: params.transcripts_dir = \"./transcripts/\"" - def transcriptsDir = - params.containsKey("transcripts_dir") ? params.transcripts_dir : - params.containsKey("transcriptsDir") ? params.transcriptsDir : - params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : - params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : - null - if (transcriptsDir != null) { - def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') - def transcriptsPublishDir = [ - path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", - saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", - mode: "copy" - ] - def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] - workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir - } - } - - // if this is a stubrun, remove certain directives? - if (workflow.stubRun) { - workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) - } - - for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { - assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" - } - } - - // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? - for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { - log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." - } - } - - // check fromState - workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) - - // check toState - workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) - - // return output - return workflowArgs -} - -def _processFromState(fromState, key_, config_) { - assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : - "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" - if (fromState == null) { - return null - } - - // if fromState is a List, convert to map - if (fromState instanceof List) { - // check whether fromstate is a list[string] - assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" - fromState = fromState.collectEntries{[it, it]} - } - - // if fromState is a map, convert to closure - if (fromState instanceof Map) { - // check whether fromstate is a map[string, string] - assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" - assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" - def fromStateMap = fromState.clone() - def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} - // turn the map into a closure to be used later on - fromState = { it -> - def state = it[1] - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def data = fromStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (state.containsKey(origkey)) { - [[newkey, state[origkey]]] - } else if (!requiredInputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") - } - }.collectEntries() - data - } - } - - return fromState -} - -def _processToState(toState, key_, config_) { - if (toState == null) { - toState = { tup -> tup[1] } - } - - // toState should be a closure, map[string, string], or list[string] - assert toState instanceof Closure || toState instanceof Map || toState instanceof List : - "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" - - // if toState is a List, convert to map - if (toState instanceof List) { - // check whether toState is a list[string] - assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" - toState = toState.collectEntries{[it, it]} - } - - // if toState is a map, convert to closure - if (toState instanceof Map) { - // check whether toState is a map[string, string] - assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" - assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" - def toStateMap = toState.clone() - def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} - // turn the map into a closure to be used later on - toState = { it -> - def output = it[1] - def state = it[2] - assert output instanceof Map : "Error in module '$key_': the output is not a Map" - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def extraEntries = toStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (output.containsKey(origkey)) { - [[newkey, output[origkey]]] - } else if (!requiredOutputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") - } - }.collectEntries() - state + extraEntries - } - } - - return toState -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' -def _debug(workflowArgs, debugKey) { - if (workflowArgs.debug) { - view { "process '${workflowArgs.key}' $debugKey tuple: $it" } - } else { - map { it } - } -} - -// depends on: innerWorkflowFactory -def workflowFactory(Map args, Map defaultWfArgs, Map meta) { - def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) - def key_ = workflowArgs["key"] - def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} - - workflow workflowInstance { - take: input_ - - main: - def chModified = input_ - | checkUniqueIds([:]) - | _debug(workflowArgs, "input") - | map { tuple -> - tuple = deepClone(tuple) - - if (workflowArgs.map) { - tuple = workflowArgs.map(tuple) - } - if (workflowArgs.mapId) { - tuple[0] = workflowArgs.mapId(tuple[0]) - } - if (workflowArgs.mapData) { - tuple[1] = workflowArgs.mapData(tuple[1]) - } - if (workflowArgs.mapPassthrough) { - tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) - } - - // check tuple - assert tuple instanceof List : - "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - assert tuple.size() >= 2 : - "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: tuple.size() == ${tuple.size()}" - - // check id field - if (tuple[0] instanceof GString) { - tuple[0] = tuple[0].toString() - } - assert tuple[0] instanceof CharSequence : - "Error in module '${key_}': first element of tuple in channel should be a String\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: ${tuple[0]}" - - // match file to input file - if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { - def inputFiles = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - - assert inputFiles.size() == 1 : - "Error in module '${key_}' id '${tuple[0]}'.\n" + - " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + - " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" - - tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() - } - - // check data field - assert tuple[1] instanceof Map : - "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // rename keys of data field in tuple - if (workflowArgs.renameKeys) { - assert workflowArgs.renameKeys instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" - assert tuple[1] instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // TODO: allow renameKeys to be a function? - workflowArgs.renameKeys.each { newKey, oldKey -> - assert newKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" - assert oldKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" - assert tuple[1].containsKey(oldKey) : - "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + - " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" - tuple[1].put(newKey, tuple[1][oldKey]) - } - tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) - } - tuple - } - - - def chRun = null - def chPassthrough = null - if (workflowArgs.runIf) { - def runIfBranch = chModified.branch{ tup -> - run: workflowArgs.runIf(tup[0], tup[1]) - passthrough: true - } - chRun = runIfBranch.run - chPassthrough = runIfBranch.passthrough - } else { - chRun = chModified - chPassthrough = Channel.empty() - } - - def chRunFiltered = workflowArgs.filter ? - chRun | filter{workflowArgs.filter(it)} : - chRun - - def chArgs = workflowArgs.fromState ? - chRunFiltered | map{ - def new_data = workflowArgs.fromState(it.take(2)) - [it[0], new_data] - } : - chRunFiltered | map {tup -> tup.take(2)} - - // fill in defaults - def chArgsWithDefaults = chArgs - | map { tuple -> - def id_ = tuple[0] - def data_ = tuple[1] - - // TODO: could move fromState to here - - // fetch default params from functionality - def defaultArgs = meta.config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - // fetch overrides in params - def paramArgs = meta.config.allArguments - .findAll { par -> - def argKey = key_ + "__" + par.plainName - params.containsKey(argKey) - } - .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } - - // fetch overrides in data - def dataArgs = meta.config.allArguments - .findAll { data_.containsKey(it.plainName) } - .collectEntries { [ it.plainName, data_[it.plainName] ] } - - // combine params - def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs - - // remove arguments with explicit null values - combinedArgs - .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} - - combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) - - [id_, combinedArgs] + tuple.drop(2) - } - - // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutputMulti = chArgsWithDefaults - | _debug(workflowArgs, "processed") - // run workflow - | innerWorkflowFactory(workflowArgs) - def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] - assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" - // Add a channel ID to the events, which designates the channel the event was emitted from as a running number - // This number is used to sort the events later when the events are gathered from across the channels. - def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> - def newChannel = channel - | map {tuple -> - assert tuple instanceof List : - "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - - def newEvent = [channelIndex] + tuple - return newEvent - } - return newChannel - } - // Put the events into 1 channel, cover case where there is only one channel is emitted - def chInitialOutput = chInitialOutputList.size() > 1 ? \ - chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ - chInitialOutputListWithIndexedEvents[0] - def chInitialOutputProcessed = chInitialOutput - | map { tuple -> - def channelId = tuple[0] - def id_ = tuple[1] - def output_ = tuple[2] - - // see if output map contains metadata - def meta_ = - output_ instanceof Map && output_.containsKey("_meta") ? - output_["_meta"] : - [:] - def join_id = meta_.join_id ?: id_ - - // remove metadata - output_ = output_.findAll{k, v -> k != "_meta"} - - // check value types - output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - - [join_id, channelId, id_, output_] - } - // | view{"chInitialOutput: ${it.take(3)}"} - - // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] - def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) - // input tuple format: [join_id, channel_id, id, output, prev_state, ...] - // output tuple format: [join_id, channel_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(2).take(3)) - tup.take(3) + [new_state] + tup.drop(5) - } - if (workflowArgs.auto.publish == "state") { - def chPublishFiles = chPublishWithPreviousState - // input tuple format: [join_id, channel_id, id, new_state, ...] - // output tuple format: [join_id, channel_id, id, new_state] - | map{ tup -> - tup.take(4) - } - - safeJoin(chPublishFiles, chArgsWithDefaults, key_) - // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(2).take(3) - } - | publishFilesByConfig(key: key_, config: meta.config) - } - // Join the state from the events that were emitted from different channels - def chJoined = chInitialOutputProcessed - | map {tuple -> - def join_id = tuple[0] - def channel_id = tuple[1] - def id = tuple[2] - def other = tuple.drop(3) - // Below, groupTuple is used to join the events. To make sure resuming a workflow - // keeps working, the output state must be deterministic. This means the state needs to be - // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', - // but hashing the state when it is large can be problematic in terms of performance. - // Therefore, a custom comparator function is provided. We add the channel ID to the - // states so that we can use the channel ID to sort the items. - def stateWithChannelID = [[channel_id] * other.size(), other].transpose() - // A comparator that is provided to groupTuple's 'sort' argument is applied - // to all elements of the event tuple (that is not the 'id'). The comparator - // closure that is used below expects the input to be List. So the join_id and - // channel_id must also be wrapped in a list. - [[join_id], [channel_id], id] + stateWithChannelID - } - | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) - | map {join_ids, _, id, statesWithChannelID -> - // Remove the channel IDs from the states - def states = statesWithChannelID.collect{it[1]} - def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} - assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." - def newJoinIdUnique = newJoinId[0] - - // Merge the states from the different channels - def newState = states.inject([:]){ old_state, state_to_add -> - return old_state + state_to_add.collectEntries{k, v -> - if (!multipleArgs.contains(k)) { - // if the key is not a multiple argument, we expect only one value - if (old_state.containsKey(k)) { - assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." - } - [k, v] - } else { - // if the key is a multiple argument, append the different values into one list - def prevValue = old_state.getOrDefault(k, []) - def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] - [k, prevValueAsList + v] - } - } - } - - _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) - - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { - newState = newState.values()[0] - } - - return [newJoinIdUnique, id, newState] - } - - // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chJoined, chRunFiltered, key_) - // input tuple format: [join_id, id, output, prev_state, ...] - // output tuple format: [join_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(1).take(3)) - tup.take(2) + [new_state] + tup.drop(4) - } - - if (workflowArgs.auto.publish == "state") { - def chPublishStates = chNewState - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [join_id, id, new_state] - | map{ tup -> - tup.take(3) - } - - safeJoin(chPublishStates, chArgsWithDefaults, key_) - // input tuple format: [join_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(1).take(3) - } - | publishStatesByConfig(key: key_, config: meta.config) - } - chReturn = chNewState - | map { tup -> - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [id, new_state, ...] - tup.drop(1) - } - | _debug(workflowArgs, "output") - | concat(chPassthrough) - - emit: chReturn - } - - def wf = workflowInstance.cloneWithName(key_) - - // add factory function - wf.metaClass.run = { runArgs -> - workflowFactory(runArgs, workflowArgs, meta) - } - // add config to module for later introspection - wf.metaClass.config = meta.config - - return wf -} - -nextflow.enable.dsl=2 - -// START COMPONENT-SPECIFIC CODE - -// create meta object -meta = [ - "resources_dir": moduleDir.toRealPath().normalize(), - "config": processConfig(readJsonBlob('''{ - "name" : "grep_annotation_column", - "namespace" : "metadata", - "version" : "disable-scrublet_build", - "authors" : [ - { - "name" : "Dries Schaumont", - "roles" : [ - "maintainer" - ], - "info" : { - "role" : "Core Team Member", - "links" : { - "email" : "dries@data-intuitive.com", - "github" : "DriesSchaumont", - "orcid" : "0000-0002-4389-0440", - "linkedin" : "dries-schaumont" - }, - "organizations" : [ - { - "name" : "Data Intuitive", - "href" : "https://www.data-intuitive.com", - "role" : "Data Scientist" - } - ] - } - } - ], - "argument_groups" : [ - { - "name" : "Inputs", - "description" : "Arguments related to the input dataset.", - "arguments" : [ - { - "type" : "file", - "name" : "--input", - "alternatives" : [ - "-i" - ], - "description" : "Path to the input .h5mu.", - "example" : [ - "sample_path" - ], - "must_exist" : true, - "create_parent" : true, - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--input_column", - "description" : "Column to query. If not specified, use .var_names or .obs_names, depending on the value of --matrix", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--input_layer", - "description" : "Input data to use when calculating fraction of observations that match with the query. \nOnly used when --output_fraction_column is provided. If not specified, .X is used.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--modality", - "description" : "Which modality to get the annotation matrix from.\n", - "example" : [ - "rna" - ], - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--matrix", - "description" : "Matrix to fetch the column from that will be searched.", - "example" : [ - "var" - ], - "required" : false, - "choices" : [ - "var", - "obs" - ], - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Outputs", - "description" : "Arguments related to how the output will be written.", - "arguments" : [ - { - "type" : "file", - "name" : "--output", - "alternatives" : [ - "-o" - ], - "description" : "Location of the output MuData file.\n", - "example" : [ - "output.h5mu" - ], - "must_exist" : true, - "create_parent" : true, - "required" : false, - "direction" : "output", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_match_column", - "description" : "Name of the column to write the result to.", - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_fraction_column", - "description" : "For the opposite axis, name of the column to write the fraction of \nobservations that matches to the pattern.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_compression", - "description" : "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", - "example" : [ - "gzip" - ], - "required" : false, - "choices" : [ - "gzip", - "lzf" - ], - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Query options", - "description" : "Options related to the query", - "arguments" : [ - { - "type" : "string", - "name" : "--regex_pattern", - "description" : "Regex to use to match with the input column.", - "example" : [ - "^[mM][tT]-" - ], - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - } - ], - "resources" : [ - { - "type" : "python_script", - "path" : "script.py", - "is_executable" : true - }, - { - "type" : "file", - "path" : "/src/utils/setup_logger.py" - }, - { - "type" : "file", - "path" : "/src/utils/compress_h5mu.py" - }, - { - "type" : "file", - "path" : "/src/workflows/utils/labels.config", - "dest" : "nextflow_labels.config" - } - ], - "description" : "Perform a regex lookup on a column from the annotation matrices .obs or .var.\nThe annotation matrix can originate from either a modality, or all modalities (global .var or .obs).\n", - "test_resources" : [ - { - "type" : "python_script", - "path" : "test.py", - "is_executable" : true - }, - { - "type" : "file", - "path" : "/resources_test/concat_test_data/e18_mouse_brain_fresh_5k_filtered_feature_bc_matrix_subset_unique_obs.h5mu" - } - ], - "status" : "enabled", - "scope" : { - "image" : "public", - "target" : "public" - }, - "license" : "MIT", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io" - }, - "runners" : [ - { - "type" : "executable", - "id" : "executable", - "docker_setup_strategy" : "ifneedbepullelsecachedbuild" - }, - { - "type" : "nextflow", - "id" : "nextflow", - "directives" : { - "label" : [ - "singlecpu", - "lowmem" - ], - "tag" : "$id" - }, - "auto" : { - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false - }, - "config" : { - "labels" : { - "mem1gb" : "memory = 1000000000.B", - "mem2gb" : "memory = 2000000000.B", - "mem5gb" : "memory = 5000000000.B", - "mem10gb" : "memory = 10000000000.B", - "mem20gb" : "memory = 20000000000.B", - "mem50gb" : "memory = 50000000000.B", - "mem100gb" : "memory = 100000000000.B", - "mem200gb" : "memory = 200000000000.B", - "mem500gb" : "memory = 500000000000.B", - "mem1tb" : "memory = 1000000000000.B", - "mem2tb" : "memory = 2000000000000.B", - "mem5tb" : "memory = 5000000000000.B", - "mem10tb" : "memory = 10000000000000.B", - "mem20tb" : "memory = 20000000000000.B", - "mem50tb" : "memory = 50000000000000.B", - "mem100tb" : "memory = 100000000000000.B", - "mem200tb" : "memory = 200000000000000.B", - "mem500tb" : "memory = 500000000000000.B", - "mem1gib" : "memory = 1073741824.B", - "mem2gib" : "memory = 2147483648.B", - "mem4gib" : "memory = 4294967296.B", - "mem8gib" : "memory = 8589934592.B", - "mem16gib" : "memory = 17179869184.B", - "mem32gib" : "memory = 34359738368.B", - "mem64gib" : "memory = 68719476736.B", - "mem128gib" : "memory = 137438953472.B", - "mem256gib" : "memory = 274877906944.B", - "mem512gib" : "memory = 549755813888.B", - "mem1tib" : "memory = 1099511627776.B", - "mem2tib" : "memory = 2199023255552.B", - "mem4tib" : "memory = 4398046511104.B", - "mem8tib" : "memory = 8796093022208.B", - "mem16tib" : "memory = 17592186044416.B", - "mem32tib" : "memory = 35184372088832.B", - "mem64tib" : "memory = 70368744177664.B", - "mem128tib" : "memory = 140737488355328.B", - "mem256tib" : "memory = 281474976710656.B", - "mem512tib" : "memory = 562949953421312.B", - "cpu1" : "cpus = 1", - "cpu2" : "cpus = 2", - "cpu5" : "cpus = 5", - "cpu10" : "cpus = 10", - "cpu20" : "cpus = 20", - "cpu50" : "cpus = 50", - "cpu100" : "cpus = 100", - "cpu200" : "cpus = 200", - "cpu500" : "cpus = 500", - "cpu1000" : "cpus = 1000" - }, - "script" : [ - "includeConfig(\\"nextflow_labels.config\\")" - ] - }, - "debug" : false, - "container" : "docker" - } - ], - "engines" : [ - { - "type" : "docker", - "id" : "docker", - "image" : "python:3.11-slim", - "target_tag" : "disable-scrublet_build", - "namespace_separator" : "/", - "setup" : [ - { - "type" : "apt", - "packages" : [ - "procps" - ], - "interactive" : false - }, - { - "type" : "python", - "user" : false, - "packages" : [ - "anndata~=0.11.1", - "mudata~=0.3.1" - ], - "script" : [ - "exec(\\"try:\\\\n import awkward\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: exit(1)\\")" - ], - "upgrade" : true - } - ], - "test_setup" : [ - { - "type" : "apt", - "packages" : [ - "git" - ], - "interactive" : false - }, - { - "type" : "python", - "user" : false, - "packages" : [ - "viashpy==0.8.0" - ], - "github" : [ - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" - ], - "upgrade" : true - } - ] - } - ], - "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/metadata/grep_annotation_column/config.vsh.yaml", - "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/metadata/grep_annotation_column", - "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", - "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" - }, - "package_config" : { - "name" : "openpipeline", - "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", - "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", - "info" : { - "test_resources" : [ - { - "type" : "s3", - "path" : "s3://openpipelines-data", - "dest" : "resources_test" - } - ] - }, - "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", - "config_mods" : [ - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" - ], - "keywords" : [ - "single-cell", - "multimodal" - ], - "license" : "MIT", - "organization" : "openpipelines-bio", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io", - "homepage" : "https://openpipelines.bio", - "documentation" : "https://openpipelines.bio/fundamentals", - "issue_tracker" : "https://github.com/openpipelines-bio/openpipeline/issues" - } - } -}''')) -] - -// resolve dependencies dependencies (if any) - - -// inner workflow -// inner workflow hook -def innerWorkflowFactory(args) { - def rawScript = '''set -e -tempscript=".viash_script.py" -cat > "$tempscript" << VIASHMAIN -import sys -import mudata as mu -from pathlib import Path -from operator import attrgetter -from pandas import Series -import scipy as sc -import re -import numpy as np - - -### VIASH START -# The following code has been auto-generated by Viash. -par = { - 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'input_column': $( if [ ! -z ${VIASH_PAR_INPUT_COLUMN+x} ]; then echo "r'${VIASH_PAR_INPUT_COLUMN//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'input_layer': $( if [ ! -z ${VIASH_PAR_INPUT_LAYER+x} ]; then echo "r'${VIASH_PAR_INPUT_LAYER//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'matrix': $( if [ ! -z ${VIASH_PAR_MATRIX+x} ]; then echo "r'${VIASH_PAR_MATRIX//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_match_column': $( if [ ! -z ${VIASH_PAR_OUTPUT_MATCH_COLUMN+x} ]; then echo "r'${VIASH_PAR_OUTPUT_MATCH_COLUMN//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_fraction_column': $( if [ ! -z ${VIASH_PAR_OUTPUT_FRACTION_COLUMN+x} ]; then echo "r'${VIASH_PAR_OUTPUT_FRACTION_COLUMN//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_compression': $( if [ ! -z ${VIASH_PAR_OUTPUT_COMPRESSION+x} ]; then echo "r'${VIASH_PAR_OUTPUT_COMPRESSION//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'regex_pattern': $( if [ ! -z ${VIASH_PAR_REGEX_PATTERN+x} ]; then echo "r'${VIASH_PAR_REGEX_PATTERN//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ) -} -meta = { - 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) -} -dep = { - -} - -### VIASH END -sys.path.append(meta["resources_dir"]) -from setup_logger import setup_logger -from compress_h5mu import write_h5ad_to_h5mu_with_compression - -logger = setup_logger() - - -def describe_array(arr, msg): - # Note: sc.stats returns a DescribeResult NamedTuple. For NamedTuples, - # the _asdict method is public facing even though it starts with an underscore. - description = sc.stats.describe(arr)._asdict() - logger.info( - "%s:\\\\nshape: %s\\\\nmean: %s\\\\nnobs: %s\\\\n" - "variance: %s\\\\nmin: %s\\\\nmax: %s\\\\ncontains na: %s\\\\ndtype: %s\\\\ncontains 0: %s", - msg, - arr.shape, - description["mean"], - description["nobs"], - description["variance"], - description["minmax"][0], - description["minmax"][1], - np.isnan(arr).any(), - arr.dtype, - (arr == 0).any(), - ) - - -def main(par): - input_file, output_file, mod_name = ( - Path(par["input"]), - Path(par["output"]), - par["modality"], - ) - logger.info(f"Compiling regular expression '{par['regex_pattern']}'.") - try: - compiled_regex = re.compile(par["regex_pattern"]) - except (TypeError, re.error) as e: - raise ValueError( - f"{par['regex_pattern']} is not a valid regular expression pattern." - ) from e - else: - if compiled_regex.groups: - raise NotImplementedError( - "Using match groups is not supported by this component." - ) - logger.info("Reading input file %s, modality %s.", input_file, mod_name) - - modality_data = mu.read_h5ad(input_file, mod=mod_name) - logger.info("Reading input file done.") - logger.info("Using annotation dataframe '%s'.", par["matrix"]) - annotation_matrix = getattr(modality_data, par["matrix"]) - default_column = {"var": attrgetter("var_names"), "obs": attrgetter("obs_names")} - if par["input_column"]: - logger.info("Input column '%s' was specified.", par["input_column"]) - try: - annotation_column = annotation_matrix[par["input_column"]] - except KeyError as e: - raise ValueError( - f"Column {par['input_column']} could not be found for modality " - f"{par['modality']}. Available columns:" - f" {','.join(annotation_matrix.columns.to_list())}" - ) from e - else: - logger.info(f"No input column specified, using '.{par['matrix']}_names'") - annotation_column = default_column[par["matrix"]](modality_data).to_series() - logger.info("Applying regex search.") - grep_result = annotation_column.str.contains(par["regex_pattern"], regex=True) - logger.info("Search results: %s", grep_result.value_counts()) - # A Series object cannot be used as an indexer for a scipy sparse array - # when the data type is a pandas boolean extension array because - # extension arrays do not define .nonzero() - # See https://github.com/pandas-dev/pandas/issues/46025 - grep_result = grep_result.to_numpy(dtype="bool", na_value=False) - - other_axis_attribute = {"var": "obs", "obs": "var"} - if par["output_fraction_column"]: - logger.info( - "Enabled writing the fraction of values that matches to the pattern." - ) - input_layer = ( - modality_data.X - if not par["input_layer"] - else modality_data.layers[par["input_layer"]] - ) - totals = np.ravel(input_layer.sum(axis=1)) - describe_array(totals, "Summary of total counts for layer") - counts_for_matches = np.ravel(input_layer[:, grep_result].sum(axis=1)) - describe_array(counts_for_matches, "Summary of counts matching grep") - with np.errstate(all="raise"): - pct_matching = np.divide( - counts_for_matches, - totals, - out=np.zeros_like(totals, dtype=np.float64), - where=(~np.isclose(totals, np.zeros_like(totals))), - ) - logger.info("Testing wether or not fractions data contains NA.") - assert ~np.isnan(pct_matching).any(), "Fractions should not contain NA." - logger.info("Fraction statistics: \\\\n%s", Series(pct_matching).describe()) - pct_matching = np.where(np.isclose(pct_matching, 0, atol=1e-6), 0, pct_matching) - pct_matching = np.where(np.isclose(pct_matching, 1, atol=1e-6), 1, pct_matching) - assert (np.logical_and(pct_matching >= 0, pct_matching <= 1)).all(), ( - "Fractions are not within bounds, please report this as a bug" - ) - output_matrix = other_axis_attribute[par["matrix"]] - logger.info( - "Writing fractions to matrix '%s', column '%s'", - output_matrix, - par["output_fraction_column"], - ) - getattr(modality_data, output_matrix)[par["output_fraction_column"]] = ( - pct_matching - ) - logger.info( - "Adding values that matched the pattern to '%s', column '%s'", - par["matrix"], - par["output_match_column"], - ) - getattr(modality_data, par["matrix"])[par["output_match_column"]] = grep_result - logger.info( - "Writing out data to '%s' with compression '%s'.", - output_file, - par["output_compression"], - ) - write_h5ad_to_h5mu_with_compression( - output_file, par["input"], mod_name, modality_data, par["output_compression"] - ) - - -if __name__ == "__main__": - main(par) -VIASHMAIN -python -B "$tempscript" -''' - - return vdsl3WorkflowFactory(args, meta, rawScript) -} - - - -/** - * Generate a workflow for VDSL3 modules. - * - * This function is called by the workflowFactory() function. - * - * Input channel: [id, input_map] - * Output channel: [id, output_map] - * - * Internally, this workflow will convert the input channel - * to a format which the Nextflow module will be able to handle. - */ -def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { - def key = args["key"] - def processObj = null - - workflow processWf { - take: input_ - main: - - if (processObj == null) { - processObj = _vdsl3ProcessFactory(args, meta, rawScript) - } - - output_ = input_ - | map { tuple -> - def id = tuple[0] - def data_ = tuple[1] - - if (workflow.stubRun) { - // add id if missing - data_ = [id: 'stub'] + data_ - } - - // process input files separately - def inputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - .collect { par -> - def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] - def inputFiles = [] - if (val == null) { - inputFiles = [] - } else if (val instanceof List) { - inputFiles = val - } else if (val instanceof Path) { - inputFiles = [ val ] - } else { - inputFiles = [] - } - if (!workflow.stubRun) { - // throw error when an input file doesn't exist - inputFiles.each{ file -> - assert file.exists() : - "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + - " Required input file does not exist.\n" + - " Path: '$file'.\n" + - " Expected input file to exist" - } - } - inputFiles - } - - // remove input files - def argsExclInputFiles = meta.config.allArguments - .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } - .collectEntries { par -> - def parName = par.plainName - def val = data_[parName] - if (par.multiple && val instanceof Collection) { - val = val.join(par.multiple_sep) - } - if (par.direction == "output" && par.type == "file") { - val = val - .replaceAll('\\$id', id) - .replaceAll('\\$\\{id\\}', id) - .replaceAll('\\$key', key) - .replaceAll('\\$\\{key\\}', key) - } - [parName, val] - } - - [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] - } - | processObj - | map { output -> - def outputFiles = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .indexed() - .collectEntries{ index, par -> - def out = output[index + 1] - // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) - if (!out instanceof List || out.size() <= 1) { - if (par.multiple) { - out = [] - } else { - assert !par.required : - "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + - " Required output file is missing" - out = null - } - } else if (out.size() == 2 && !par.multiple) { - out = out[1] - } else { - out = out.drop(1) - } - [ par.plainName, out ] - } - - // drop null outputs - outputFiles.removeAll{it.value == null} - - [ output[0], outputFiles ] - } - emit: output_ - } - - return processWf -} - -// depends on: session? -def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { - // autodetect process key - def wfKey = workflowArgs["key"] - def procKeyPrefix = "${wfKey}_process" - def scriptMeta = nextflow.script.ScriptMeta.current() - def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} - def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} - def newNumber = (numbers + [-1]).max() + 1 - - def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" - - if (newNumber > 0) { - log.warn "Key for module '${wfKey}' is duplicated.\n", - "If you run a component multiple times in the same workflow,\n" + - "it's recommended you set a unique key for every call,\n" + - "for example: ${wfKey}.run(key: \"foo\")." - } - - // subset directives and convert to list of tuples - def drctv = workflowArgs.directives - - // TODO: unit test the two commands below - // convert publish array into tags - def valueToStr = { val -> - // ignore closures - if (val instanceof CharSequence) { - if (!val.matches('^[{].*[}]$')) { - '"' + val + '"' - } else { - val - } - } else if (val instanceof List) { - "[" + val.collect{valueToStr(it)}.join(", ") + "]" - } else if (val instanceof Map) { - "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" - } else { - val.inspect() - } - } - - // multiple entries allowed: label, publishdir - def drctvStrs = drctv.collect { key, value -> - if (key in ["label", "publishDir"]) { - value.collect{ val -> - if (val instanceof Map) { - "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") - } else if (val == null) { - "" - } else { - "\n$key " + valueToStr(val) - } - }.join() - } else if (value instanceof Map) { - "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") - } else { - "\n$key " + valueToStr(value) - } - }.join() - - def inputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } - .join() - - def outputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .collect { par -> - // insert dummy into every output (see nextflow-io/nextflow#2678) - if (!par.multiple) { - ', path{[".exitcode", args.' + par.plainName + ']}' - } else { - ', path{[".exitcode"] + args.' + par.plainName + '}' - } - } - .join() - - // TODO: move this functionality somewhere else? - if (workflowArgs.auto.transcript) { - outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' - } else { - outputPaths = outputPaths + ', path{[".exitcode"]}' - } - - // create dirs for output files (based on BashWrapper.createParentFiles) - def createParentStr = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" && it.create_parent } - .collect { par -> - def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" - "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" - } - .join("\n") - - // construct inputFileExports - def inputFileExports = meta.config.allArguments - .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } - .collect { par -> - def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" - "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" - } - - // NOTE: if using docker, use /tmp instead of tmpDir! - def tmpDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('VIASH_TMPDIR') ?: - System.getenv('VIASH_TEMPDIR') ?: - System.getenv('VIASH_TMP') ?: - System.getenv('TEMP') ?: - System.getenv('TMPDIR') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMP') ?: - '/tmp' - ).toAbsolutePath() - - // construct stub - def stub = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .collect { par -> - "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" - } - .join("\n") - - // escape script - def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') - - // publishdir assert - def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? - """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : - "" - - // generate process string - def procStr = - """nextflow.enable.dsl=2 - | - |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } - |process $procKey {$drctvStrs - |input: - | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") - |output: - | tuple val("\$id")$outputPaths, optional: true - |stub: - |\"\"\" - |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } - |$stub - |\"\"\" - |script:$assertStr - |def parInject = args - | .findAll{key, value -> value != null} - | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} - | .join("\\n") - |\"\"\" - |# meta exports - |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" - |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" - |export VIASH_META_NAME="${meta.config.name}" - |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" - |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" - |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } - |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } - |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then - | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) - | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) - | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) - | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) - | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) - | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) - | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) - | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) - | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) - | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) - |fi - | - |# meta synonyms - |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" - |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" - | - |# create output dirs if need be - |function mkdir_parent { - | for file in "\\\$@"; do - | mkdir -p "\\\$(dirname "\\\$file")" - | done - |} - |$createParentStr - | - |# argument exports${inputFileExports.join()} - |\$parInject - | - |# process script - |${escapedScript} - |\"\"\" - |} - |""".stripMargin() - - // TODO: print on debug - // if (workflowArgs.debug == true) { - // println("######################\n$procStr\n######################") - // } - - // write process to temp file - def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") - addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } - tempFile.text = procStr - - // create process from temp file - def binding = new nextflow.script.ScriptBinding([:]) - def session = nextflow.Nextflow.getSession() - def parser = _getScriptLoader(session) - .setModule(true) - .setBinding(binding) - def moduleScript = parser.runScript(tempFile) - .getScript() - - // register module in meta - def module = new nextflow.script.IncludeDef.Module(name: procKey) - scriptMeta.addModule(moduleScript, module.name, module.alias) - - // retrieve and return process from meta - return scriptMeta.getProcess(procKey) -} - -// use Reflection to get a ScriptParser / ScriptLoader -// <25.02.0-edge: new nextflow.script.ScriptParser(session) -// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) -def _getScriptLoader(nextflow.Session session) { - // try using the old method - try { - Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') - return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) - } catch (ClassNotFoundException e) { - // else try with the new method - try { - Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') - def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) - return createMethod.invoke(null, session) // null because create is static - } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { - // Handle the case where neither class is found - throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) - } - } -} - -// defaults -meta["defaults"] = [ - // key to be used to trace the process and determine output names - key: null, - - // fixed arguments to be passed to script - args: [:], - - // default directives - directives: readJsonBlob('''{ - "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/metadata/grep_annotation_column", - "tag" : "disable-scrublet_build" - }, - "label" : [ - "singlecpu", - "lowmem" - ], - "tag" : "$id" -}'''), - - // auto settings - auto: readJsonBlob('''{ - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false -}'''), - - // Apply a map over the incoming tuple - // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` - map: null, - - // Apply a map over the ID element of a tuple (i.e. the first element) - // Example: `{ id -> id + "_foo" }` - mapId: null, - - // Apply a map over the data element of a tuple (i.e. the second element) - // Example: `{ data -> [ input: data.output ] }` - mapData: null, - - // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) - // Example: `{ pt -> pt.drop(1) }` - mapPassthrough: null, - - // Filter the channel - // Example: `{ tup -> tup[0] == "foo" }` - filter: null, - - // Choose whether or not to run the component on the tuple if the condition is true. - // Otherwise, the tuple will be passed through. - // Example: `{ tup -> tup[0] != "skip_this" }` - runIf: null, - - // Rename keys in the data field of the tuple (i.e. the second element) - // Will likely be deprecated in favour of `fromState`. - // Example: `[ "new_key": "old_key" ]` - renameKeys: null, - - // Fetch data from the state and pass it to the module without altering the current state. - // - // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be passed to the module as is. - // - If it is a `List[String]`, the data will be the values of the state at the given keys. - // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. - // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. - // - // Example: `{ id, state -> [input: state.fastq_file] }` - // Default: `null` - fromState: null, - - // Determine how the state should be updated after the module has been run. - // - // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be replaced with the output of the module. - // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. - // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. - // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. - // - // Example: `{ id, output, state -> state + [counts: state.output] }` - // Default: `{ id, output, state -> output }` - toState: null, - - // Whether or not to print debug messages - // Default: `false` - debug: false -] - -// initialise default workflow -meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) - -// add workflow to environment -nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) - -// anonymous workflow for running this module as a standalone -workflow { - // add id argument if it's not already in the config - // TODO: deep copy - def newConfig = deepClone(meta.config) - def newParams = deepClone(params) - - def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} - if (!argsContainsId) { - def idArg = [ - 'name': '--id', - 'required': false, - 'type': 'string', - 'description': 'A unique id for every entry.', - 'multiple': false - ] - newConfig.arguments.add(0, idArg) - newConfig = processConfig(newConfig) - } - if (!newParams.containsKey("id")) { - newParams.id = "run" - } - - helpMessage(newConfig) - - channelFromParams(newParams, newConfig) - // make sure id is not in the state if id is not in the args - | map {id, state -> - if (!argsContainsId) { - [id, state.findAll{k, v -> k != "id"}] - } else { - [id, state] - } - } - | meta.workflow.run( - auto: [ publish: "state" ] - ) -} - -// END COMPONENT-SPECIFIC CODE diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow.config deleted file mode 100644 index 212e1dc..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow.config +++ /dev/null @@ -1,126 +0,0 @@ -manifest { - name = 'metadata/grep_annotation_column' - mainScript = 'main.nf' - nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' - description = 'Perform a regex lookup on a column from the annotation matrices .obs or .var.\nThe annotation matrix can originate from either a modality, or all modalities (global .var or .obs).\n' - author = 'Dries Schaumont' -} - -process.container = 'nextflow/bash:latest' - -// detect tempdir -tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' -).toAbsolutePath() - -profiles { - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - docker { - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - singularity { - singularity.enabled = true - singularity.autoMounts = true - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - podman { - podman.enabled = true - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - shifter { - shifter.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - } - charliecloud { - charliecloud.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - } -} - -process{ - withLabel: mem1gb { memory = 1000000000.B } - withLabel: mem2gb { memory = 2000000000.B } - withLabel: mem5gb { memory = 5000000000.B } - withLabel: mem10gb { memory = 10000000000.B } - withLabel: mem20gb { memory = 20000000000.B } - withLabel: mem50gb { memory = 50000000000.B } - withLabel: mem100gb { memory = 100000000000.B } - withLabel: mem200gb { memory = 200000000000.B } - withLabel: mem500gb { memory = 500000000000.B } - withLabel: mem1tb { memory = 1000000000000.B } - withLabel: mem2tb { memory = 2000000000000.B } - withLabel: mem5tb { memory = 5000000000000.B } - withLabel: mem10tb { memory = 10000000000000.B } - withLabel: mem20tb { memory = 20000000000000.B } - withLabel: mem50tb { memory = 50000000000000.B } - withLabel: mem100tb { memory = 100000000000000.B } - withLabel: mem200tb { memory = 200000000000000.B } - withLabel: mem500tb { memory = 500000000000000.B } - withLabel: mem1gib { memory = 1073741824.B } - withLabel: mem2gib { memory = 2147483648.B } - withLabel: mem4gib { memory = 4294967296.B } - withLabel: mem8gib { memory = 8589934592.B } - withLabel: mem16gib { memory = 17179869184.B } - withLabel: mem32gib { memory = 34359738368.B } - withLabel: mem64gib { memory = 68719476736.B } - withLabel: mem128gib { memory = 137438953472.B } - withLabel: mem256gib { memory = 274877906944.B } - withLabel: mem512gib { memory = 549755813888.B } - withLabel: mem1tib { memory = 1099511627776.B } - withLabel: mem2tib { memory = 2199023255552.B } - withLabel: mem4tib { memory = 4398046511104.B } - withLabel: mem8tib { memory = 8796093022208.B } - withLabel: mem16tib { memory = 17592186044416.B } - withLabel: mem32tib { memory = 35184372088832.B } - withLabel: mem64tib { memory = 70368744177664.B } - withLabel: mem128tib { memory = 140737488355328.B } - withLabel: mem256tib { memory = 281474976710656.B } - withLabel: mem512tib { memory = 562949953421312.B } - withLabel: cpu1 { cpus = 1 } - withLabel: cpu2 { cpus = 2 } - withLabel: cpu5 { cpus = 5 } - withLabel: cpu10 { cpus = 10 } - withLabel: cpu20 { cpus = 20 } - withLabel: cpu50 { cpus = 50 } - withLabel: cpu100 { cpus = 100 } - withLabel: cpu200 { cpus = 200 } - withLabel: cpu500 { cpus = 500 } - withLabel: cpu1000 { cpus = 1000 } -} - -includeConfig("nextflow_labels.config") diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_params.yaml deleted file mode 100644 index 0669fca..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_params.yaml +++ /dev/null @@ -1,21 +0,0 @@ -# Inputs -input: # please fill in - example: "sample_path" -# input_column: "foo" -# input_layer: "foo" -modality: # please fill in - example: "rna" -# matrix: "var" - -# Outputs -# output: "$id.$key.output.h5mu" -output_match_column: # please fill in - example: "foo" -# output_fraction_column: "foo" -# output_compression: "gzip" - -# Query options -regex_pattern: # please fill in - example: "^[mM][tT]-" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_schema.json b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_schema.json deleted file mode 100644 index 1a12fe5..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/nextflow_schema.json +++ /dev/null @@ -1,131 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "grep_annotation_column", - "description": "Perform a regex lookup on a column from the annotation matrices .obs or .var.\nThe annotation matrix can originate from either a modality, or all modalities (global .var or .obs).\n", - "type": "object", - "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, - "inputs": { - "title": "Inputs", - "type": "object", - "description": "Arguments related to the input dataset.", - "properties": { - "input": { - "type": "string", - "format": "path", - "exists": true, - "description": "Path to the input .h5mu.", - "help_text": "Type: `file`, multiple: `False`, required, direction: `input`, example: `\"sample_path\"`. " - }, - "input_column": { - "type": "string", - "description": "Column to query", - "help_text": "Type: `string`, multiple: `False`. " - }, - "input_layer": { - "type": "string", - "description": "Input data to use when calculating fraction of observations that match with the query", - "help_text": "Type: `string`, multiple: `False`. " - }, - "modality": { - "type": "string", - "description": "Which modality to get the annotation matrix from.\n", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"rna\"`. " - }, - "matrix": { - "type": "string", - "description": "Matrix to fetch the column from that will be searched.", - "help_text": "Type: `string`, multiple: `False`, example: `\"var\"`, choices: ``var`, `obs``. ", - "enum": [ - "var", - "obs" - ] - } - } - }, - "outputs": { - "title": "Outputs", - "type": "object", - "description": "Arguments related to how the output will be written.", - "properties": { - "output": { - "type": "string", - "format": "path", - "description": "Location of the output MuData file.\n", - "help_text": "Type: `file`, multiple: `False`, default: `\"$id.$key.output.h5mu\"`, direction: `output`, example: `\"output.h5mu\"`. ", - "default": "$id.$key.output.h5mu" - }, - "output_match_column": { - "type": "string", - "description": "Name of the column to write the result to.", - "help_text": "Type: `string`, multiple: `False`, required. " - }, - "output_fraction_column": { - "type": "string", - "description": "For the opposite axis, name of the column to write the fraction of \nobservations that matches to the pattern.\n", - "help_text": "Type: `string`, multiple: `False`. " - }, - "output_compression": { - "type": "string", - "description": "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", - "help_text": "Type: `string`, multiple: `False`, example: `\"gzip\"`, choices: ``gzip`, `lzf``. ", - "enum": [ - "gzip", - "lzf" - ] - } - } - }, - "query options": { - "title": "Query options", - "type": "object", - "description": "Options related to the query", - "properties": { - "regex_pattern": { - "type": "string", - "description": "Regex to use to match with the input column.", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"^[mM][tT]-\"`. " - } - } - }, - "nextflow input-output arguments": { - "title": "Nextflow input-output arguments", - "type": "object", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "properties": { - "publish_dir": { - "type": "string", - "description": "Path to an output directory.", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " - } - } - } - }, - "allOf": [ - { - "$ref": "#/$defs/inputs" - }, - { - "$ref": "#/$defs/outputs" - }, - { - "$ref": "#/$defs/query options" - }, - { - "$ref": "#/$defs/nextflow input-output arguments" - } - ] -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_params.yaml deleted file mode 100644 index 3ca2dc3..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_params.yaml +++ /dev/null @@ -1,14 +0,0 @@ -# MuData Input -input: # please fill in - example: "input.h5mu" -modality: "rna" -obsm_key: # please fill in - example: "foo" - -# MuData Output -# output: "$id.$key.output.h5mu" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_params.yaml deleted file mode 100644 index de4f65f..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_params.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -obsm_input: "X_pca" -# output: "$id.$key.output.h5mu" -uns_output: "neighbors" -obsp_distances: "distances" -obsp_connectivities: "connectivities" -metric: "euclidean" -num_neighbors: 15 -seed: 0 -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml deleted file mode 100644 index 745a9fb..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml +++ /dev/null @@ -1,380 +0,0 @@ -name: "calculate_qc_metrics" -namespace: "qc" -version: "disable-scrublet_build" -authors: -- name: "Dries Schaumont" - roles: - - "author" - info: - role: "Core Team Member" - links: - email: "dries@data-intuitive.com" - github: "DriesSchaumont" - orcid: "0000-0002-4389-0440" - linkedin: "dries-schaumont" - organizations: - - name: "Data Intuitive" - href: "https://www.data-intuitive.com" - role: "Data Scientist" -argument_groups: -- name: "Inputs" - arguments: - - type: "file" - name: "--input" - description: "Input h5mu file" - info: null - example: - - "input.h5mu" - must_exist: true - create_parent: true - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--modality" - description: "Which modality from the input MuData file to process. \n" - info: null - default: - - "rna" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--layer" - description: "Layer from modality to use as input data. If not provided the .X\ - \ attribute is used.\n" - info: null - example: - - "raw_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Metrics added to .obs" - arguments: - - type: "string" - name: "--var_qc_metrics" - description: "Keys to select a boolean (containing only True or False) column\ - \ from .var.\nFor each cell, calculate the proportion of total values for genes\ - \ which are labeled 'True', \ncompared to the total sum of the values for all\ - \ genes.\n" - info: null - example: - - "ercc,highly_variable,mitochondrial" - required: false - direction: "input" - multiple: true - multiple_sep: ";" - - type: "boolean" - name: "--var_qc_metrics_fill_na_value" - description: "Fill any 'NA' values found in the columns specified with --var_qc_metrics\ - \ to 'True' or 'False'.\nas False.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "integer" - name: "--top_n_vars" - description: "Number of top vars to be used to calculate cumulative proportions.\n\ - If not specified, proportions are not calculated. `--top_n_vars 20;50` finds\n\ - cumulative proportion to the 20th and 50th most expressed vars.\n" - info: null - required: false - direction: "input" - multiple: true - multiple_sep: ";" - - type: "string" - name: "--output_obs_num_nonzero_vars" - description: "Name of column in .obs describing, for each observation, the number\ - \ of stored values\n(including explicit zeroes). In other words, the name of\ - \ the column that counts\nfor each row the number of columns that contain data.\n" - info: null - default: - - "num_nonzero_vars" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_obs_total_counts_vars" - description: "Name of the column for .obs describing, for each observation (row),\n\ - the sum of the stored values in the columns.\n" - info: null - default: - - "total_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Metrics added to .var" - arguments: - - type: "string" - name: "--output_var_num_nonzero_obs" - description: "Name of column describing, for each feature, the number of stored\ - \ values\n(including explicit zeroes). In other words, the name of the column\ - \ that counts\nfor each column the number of rows that contain data.\n" - info: null - default: - - "num_nonzero_obs" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_total_counts_obs" - description: "Name of the column in .var describing, for each feature (column),\n\ - the sum of the stored values in the rows.\n" - info: null - default: - - "total_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_obs_mean" - description: "Name of the column in .obs providing the mean of the values in each\ - \ row.\n" - info: null - default: - - "obs_mean" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_pct_dropout" - description: "Name of the column in .obs providing for each feature the percentage\ - \ of\nobservations the feature does not appear on (i.e. is missing). Same as\ - \ `--num_nonzero_obs`\nbut percentage based.\n" - info: null - default: - - "pct_dropout" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Outputs" - arguments: - - type: "file" - name: "--output" - description: "Output h5mu file." - info: null - example: - - "output.h5mu" - must_exist: true - create_parent: true - required: false - direction: "output" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_compression" - description: "Compression format to use for the output AnnData and/or Mudata objects.\n\ - By default no compression is applied.\n" - info: null - example: - - "gzip" - required: false - choices: - - "gzip" - - "lzf" - direction: "input" - multiple: false - multiple_sep: ";" -resources: -- type: "python_script" - path: "script.py" - is_executable: true -- type: "file" - path: "setup_logger.py" -- type: "file" - path: "compress_h5mu.py" -- type: "file" - path: "nextflow_labels.config" - dest: "nextflow_labels.config" -description: "Add basic quality control metrics to an .h5mu file.\n\nThe metrics are\ - \ comparable to what scanpy.pp.calculate_qc_metrics output,\nalthough they have\ - \ slightly different names:\n\nVar metrics (name in this component -> name in scanpy):\n\ - \ - pct_dropout -> pct_dropout_by_{expr_type}\n - num_nonzero_obs -> n_cells_by_{expr_type}\n\ - \ - obs_mean -> mean_{expr_type}\n - total_counts -> total_{expr_type}\n\n Obs\ - \ metrics:\n - num_nonzero_vars -> n_genes_by_{expr_type}\n - pct_{var_qc_metrics}\ - \ -> pct_{expr_type}_{qc_var}\n - total_counts_{var_qc_metrics} -> total_{expr_type}_{qc_var}\n\ - \ - pct_of_counts_in_top_{top_n_vars}_vars -> pct_{expr_type}_in_top_{n}_{var_type}\n\ - \ - total_counts -> total_{expr_type}\n \n" -test_resources: -- type: "python_script" - path: "test.py" - is_executable: true -- type: "file" - path: "pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu" -info: null -status: "enabled" -scope: - image: "public" - target: "public" -license: "MIT" -links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" -runners: -- type: "executable" - id: "executable" - docker_setup_strategy: "ifneedbepullelsecachedbuild" -- type: "nextflow" - id: "nextflow" - directives: - label: - - "singlecpu" - - "midmem" - tag: "$id" - auto: - simplifyInput: true - simplifyOutput: false - transcript: false - publish: false - config: - labels: - mem1gb: "memory = 1000000000.B" - mem2gb: "memory = 2000000000.B" - mem5gb: "memory = 5000000000.B" - mem10gb: "memory = 10000000000.B" - mem20gb: "memory = 20000000000.B" - mem50gb: "memory = 50000000000.B" - mem100gb: "memory = 100000000000.B" - mem200gb: "memory = 200000000000.B" - mem500gb: "memory = 500000000000.B" - mem1tb: "memory = 1000000000000.B" - mem2tb: "memory = 2000000000000.B" - mem5tb: "memory = 5000000000000.B" - mem10tb: "memory = 10000000000000.B" - mem20tb: "memory = 20000000000000.B" - mem50tb: "memory = 50000000000000.B" - mem100tb: "memory = 100000000000000.B" - mem200tb: "memory = 200000000000000.B" - mem500tb: "memory = 500000000000000.B" - mem1gib: "memory = 1073741824.B" - mem2gib: "memory = 2147483648.B" - mem4gib: "memory = 4294967296.B" - mem8gib: "memory = 8589934592.B" - mem16gib: "memory = 17179869184.B" - mem32gib: "memory = 34359738368.B" - mem64gib: "memory = 68719476736.B" - mem128gib: "memory = 137438953472.B" - mem256gib: "memory = 274877906944.B" - mem512gib: "memory = 549755813888.B" - mem1tib: "memory = 1099511627776.B" - mem2tib: "memory = 2199023255552.B" - mem4tib: "memory = 4398046511104.B" - mem8tib: "memory = 8796093022208.B" - mem16tib: "memory = 17592186044416.B" - mem32tib: "memory = 35184372088832.B" - mem64tib: "memory = 70368744177664.B" - mem128tib: "memory = 140737488355328.B" - mem256tib: "memory = 281474976710656.B" - mem512tib: "memory = 562949953421312.B" - cpu1: "cpus = 1" - cpu2: "cpus = 2" - cpu5: "cpus = 5" - cpu10: "cpus = 10" - cpu20: "cpus = 20" - cpu50: "cpus = 50" - cpu100: "cpus = 100" - cpu200: "cpus = 200" - cpu500: "cpus = 500" - cpu1000: "cpus = 1000" - script: - - "includeConfig(\"nextflow_labels.config\")" - debug: false - container: "docker" -engines: -- type: "docker" - id: "docker" - image: "python:3.11-slim" - target_tag: "disable-scrublet_build" - namespace_separator: "/" - setup: - - type: "apt" - packages: - - "procps" - interactive: false - - type: "python" - user: false - packages: - - "anndata~=0.11.1" - - "mudata~=0.3.1" - - "scipy" - script: - - "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\ - nelse: exit(1)\")" - upgrade: true - test_setup: - - type: "apt" - packages: - - "git" - interactive: false - - type: "python" - user: false - packages: - - "viashpy==0.8.0" - github: - - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" - upgrade: true - - type: "python" - user: false - packages: - - "scanpy" - upgrade: true - entrypoint: [] - cmd: null -build_info: - config: "src/qc/calculate_qc_metrics/config.vsh.yaml" - runner: "nextflow" - engine: "docker" - output: "target/nextflow/qc/calculate_qc_metrics" - executable: "target/nextflow/qc/calculate_qc_metrics/main.nf" - viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" - git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" -package_config: - name: "openpipeline" - summary: "Best-practice workflows for single-cell multi-omics analyses.\n" - description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ - \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ - \nIn terms of workflows, the following has been made available, but keep in mind\ - \ that\nindividual tools and functionality can be executed as standalone components\ - \ as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n\ - \ * Ingestion: Read mapping and generating a count matrix.\n * Single sample\ - \ processing: cell filtering and doublet detection.\n * Multisample processing:\ - \ Count transformation, normalization, QC metric calulations.\n * Integration:\ - \ Clustering, integration and batch correction using single and multimodal methods.\n\ - \ * Downstream analysis workflows\n" - info: - test_resources: - - type: "s3" - path: "s3://openpipelines-data" - dest: "resources_test" - viash_version: "0.9.4" - source: "src" - target: "target" - config_mods: - - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ - .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ - )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" - keywords: - - "single-cell" - - "multimodal" - license: "MIT" - organization: "openpipelines-bio" - links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" - homepage: "https://openpipelines.bio" - documentation: "https://openpipelines.bio/fundamentals" - issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/main.nf b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/main.nf deleted file mode 100644 index e171aa3..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/main.nf +++ /dev/null @@ -1,4220 +0,0 @@ -// calculate_qc_metrics disable-scrublet_build -// -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative -// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data -// Intuitive. -// -// The component may contain files which fall under a different license. The -// authors of this component should specify the license in the header of such -// files, or include a separate license file detailing the licenses of all included -// files. -// -// Component authors: -// * Dries Schaumont (author) - -//////////////////////////// -// VDSL3 helper functions // -//////////////////////////// - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' -class UnexpectedArgumentTypeException extends Exception { - String errorIdentifier - String stage - String plainName - String expectedClass - String foundClass - - // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} - UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { - super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + - "Expected type: ${expectedClass}. Found type: ${foundClass}") - this.errorIdentifier = errorIdentifier - this.stage = stage - this.plainName = plainName - this.expectedClass = expectedClass - this.foundClass = foundClass - } -} - -/** - * Checks if the given value is of the expected type. If not, an exception is thrown. - * - * @param stage The stage of the argument (input or output) - * @param par The parameter definition - * @param value The value to check - * @param errorIdentifier The identifier to use in the error message - * @return The value, if it is of the expected type - * @throws UnexpectedArgumentTypeException If the value is not of the expected type -*/ -def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { - // expectedClass will only be != null if value is not of the expected type - def expectedClass = null - def foundClass = null - - // todo: split if need be - - if (!par.required && value == null) { - expectedClass = null - } else if (par.multiple) { - if (value !instanceof Collection) { - value = [value] - } - - // split strings - value = value.collectMany{ val -> - if (val instanceof String) { - // collect() to ensure that the result is a List and not simply an array - val.split(par.multiple_sep).collect() - } else { - [val] - } - } - - // process globs - if (par.type == "file" && par.direction == "input") { - value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() - } - - // check types of elements in list - try { - value = value.collect { listVal -> - _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) - } - } catch (UnexpectedArgumentTypeException e) { - expectedClass = "List[${e.expectedClass}]" - foundClass = "List[${e.foundClass}]" - } - } else if (par.type == "string") { - // cast to string if need be. only cast if the value is a GString - if (value instanceof GString) { - value = value as String - } - expectedClass = value instanceof String ? null : "String" - } else if (par.type == "integer") { - // cast to integer if need be - if (value !instanceof Integer) { - try { - value = value as Integer - } catch (NumberFormatException e) { - expectedClass = "Integer" - } - } - } else if (par.type == "long") { - // cast to long if need be - if (value !instanceof Long) { - try { - value = value as Long - } catch (NumberFormatException e) { - expectedClass = "Long" - } - } - } else if (par.type == "double") { - // cast to double if need be - if (value !instanceof Double) { - try { - value = value as Double - } catch (NumberFormatException e) { - expectedClass = "Double" - } - } - } else if (par.type == "float") { - // cast to float if need be - if (value !instanceof Float) { - try { - value = value as Float - } catch (NumberFormatException e) { - expectedClass = "Float" - } - } - } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { - // cast to boolean if need be - if (value !instanceof Boolean) { - try { - value = value as Boolean - } catch (Exception e) { - expectedClass = "Boolean" - } - } - } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { - // cast to path if need be - if (value instanceof String) { - value = file(value, hidden: true) - } - if (value instanceof File) { - value = value.toPath() - } - expectedClass = value instanceof Path ? null : "Path" - } else if (par.type == "file" && stage == "input" && par.direction == "output") { - // cast to string if need be - if (value !instanceof String) { - try { - value = value as String - } catch (Exception e) { - expectedClass = "String" - } - } - } else { - // didn't find a match for par.type - expectedClass = par.type - } - - if (expectedClass != null) { - if (foundClass == null) { - foundClass = value.getClass().getName() - } - throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) - } - - return value -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' -Map _processInputValues(Map inputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.required && arg.direction == "input") { - assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" - } - } - - inputs = inputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" - - value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return inputs -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - outputs = outputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && it.direction == "output" } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" - - value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return outputs -} - -void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' -class IDChecker { - final def items = [] as Set - - @groovy.transform.WithWriteLock - boolean observe(String item) { - if (items.contains(item)) { - return false - } else { - items << item - return true - } - } - - @groovy.transform.WithReadLock - boolean contains(String item) { - return items.contains(item) - } - - @groovy.transform.WithReadLock - Set getItems() { - return items.clone() - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' - -/** - * Check if the ids are unique across parameter sets - * - * @param parameterSets a list of parameter sets. - */ -private void _checkUniqueIds(List>> parameterSets) { - def ppIds = parameterSets.collect{it[0]} - assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' - -// helper functions for reading params from file // -def _getChild(parent, child) { - if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { - child - } else { - def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() - parentAbsolute.replaceAll('/[^/]*$', "/") + child - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' -/** - * Figure out the param list format based on the file extension - * - * @param param_list A String containing the path to the parameter list file. - * - * @return A String containing the format of the parameter list file. - */ -def _paramListGuessFormat(param_list) { - if (param_list !instanceof String) { - "asis" - } else if (param_list.endsWith(".csv")) { - "csv" - } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { - "json" - } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { - "yaml" - } else { - "yaml_blob" - } -} - - -/** - * Read the param list - * - * @param param_list One of the following: - * - A String containing the path to the parameter list file (csv, json or yaml), - * - A yaml blob of a list of maps (yaml_blob), - * - Or a groovy list of maps (asis). - * @param config A Map of the Viash configuration. - * - * @return A List of Maps containing the parameters. - */ -def _parseParamList(param_list, Map config) { - // first determine format by extension - def paramListFormat = _paramListGuessFormat(param_list) - - def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? - file(param_list, hidden: true) : - null - - // get the correct parser function for the detected params_list format - def paramSets = [] - if (paramListFormat == "asis") { - paramSets = param_list - } else if (paramListFormat == "yaml_blob") { - paramSets = readYamlBlob(param_list) - } else if (paramListFormat == "yaml") { - paramSets = readYaml(paramListPath) - } else if (paramListFormat == "json") { - paramSets = readJson(paramListPath) - } else if (paramListFormat == "csv") { - paramSets = readCsv(paramListPath) - } else { - error "Format of provided --param_list not recognised.\n" + - "Found: '$paramListFormat'.\n" + - "Expected: a csv file, a json file, a yaml file,\n" + - "a yaml blob or a groovy list of maps." - } - - // data checks - assert paramSets instanceof List: "--param_list should contain a list of maps" - for (value in paramSets) { - assert value instanceof Map: "--param_list should contain a list of maps" - } - - // id is argument - def idIsArgument = config.allArguments.any{it.plainName == "id"} - - // Reformat from List to List> by adding the ID as first element of a Tuple2 - paramSets = paramSets.collect({ data -> - def id = data.id - if (!idIsArgument) { - data = data.findAll{k, v -> k != "id"} - } - [id, data] - }) - - // Split parameters with 'multiple: true' - paramSets = paramSets.collect({ id, data -> - data = _splitParams(data, config) - [id, data] - }) - - // The paths of input files inside a param_list file may have been specified relatively to the - // location of the param_list file. These paths must be made absolute. - if (paramListPath) { - paramSets = paramSets.collect({ id, data -> - def new_data = data.collectEntries{ parName, parValue -> - def par = config.allArguments.find{it.plainName == parName} - if (par && par.type == "file" && par.direction == "input") { - if (parValue instanceof Collection) { - parValue = parValue.collectMany{path -> - def x = _resolveSiblingIfNotAbsolute(path, paramListPath) - x instanceof Collection ? x : [x] - } - } else { - parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) - } - } - [parName, parValue] - } - [id, new_data] - }) - } - - return paramSets -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' -/** - * Split parameters for arguments that accept multiple values using their separator - * - * @param paramList A Map containing parameters to split. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A Map of parameters where the parameter values have been split into a list using - * their seperator. - */ -Map _splitParams(Map parValues, Map config){ - def parsedParamValues = parValues.collectEntries { parName, parValue -> - def parameterSettings = config.allArguments.find({it.plainName == parName}) - - if (!parameterSettings) { - // if argument is not found, do not alter - return [parName, parValue] - } - if (parameterSettings.multiple) { // Check if parameter can accept multiple values - if (parValue instanceof Collection) { - parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } - } else if (parValue instanceof String) { - parValue = parValue.split(parameterSettings.multiple_sep) - } else if (parValue == null) { - parValue = [] - } else { - parValue = [ parValue ] - } - parValue = parValue.flatten() - } - // For all parameters check if multiple values are only passed for - // arguments that allow it. Quietly simplify lists of length 1. - if (!parameterSettings.multiple && parValue instanceof Collection) { - assert parValue.size() == 1 : - "Error: argument ${parName} has too many values.\n" + - " Expected amount: 1. Found: ${parValue.size()}" - parValue = parValue[0] - } - [parName, parValue] - } - return parsedParamValues -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' -/** - * Parse nextflow parameters based on settings defined in a viash config. - * Return a list of parameter sets, each parameter set corresponding to - * an event in a nextflow channel. The output from this function can be used - * with Channel.fromList to create a nextflow channel with Vdsl3 formatted - * events. - * - * This function performs: - * - A filtering of the params which can be found in the config file. - * - Process the params_list argument which allows a user to to initialise - * a Vsdl3 channel with multiple parameter sets. Possible formats are - * csv, json, yaml, or simply a yaml_blob. A csv should have column names - * which correspond to the different arguments of this pipeline. A json or a yaml - * file should be a list of maps, each of which has keys corresponding to the - * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. - * When passing a csv, json or yaml, relative path names are relativized to the - * location of the parameter file. - * - Combine the parameter sets into a vdsl3 Channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A list of parameters with the first element of the event being - * the event ID and the second element containing a map of the parsed parameters. - */ - -private List>> _paramsToParamSets(Map params, Map config){ - // todo: fetch key from run args - def key_ = config.name - - /* parse regular parameters (not in param_list) */ - /*************************************************/ - def globalParams = config.allArguments - .findAll { params.containsKey(it.plainName) } - .collectEntries { [ it.plainName, params[it.plainName] ] } - def globalID = params.get("id", null) - - /* process params_list arguments */ - /*********************************/ - def paramList = params.containsKey("param_list") && params.param_list != null ? - params.param_list : [] - // if (paramList instanceof String) { - // paramList = [paramList] - // } - // def paramSets = paramList.collectMany{ _parseParamList(it, config) } - // TODO: be able to process param_list when it is a list of strings - def paramSets = _parseParamList(paramList, config) - if (paramSets.isEmpty()) { - paramSets = [[null, [:]]] - } - - /* combine arguments into channel */ - /**********************************/ - def processedParams = paramSets.indexed().collect{ index, tup -> - // Process ID - def id = tup[0] ?: globalID - - if (workflow.stubRun && !id) { - // if stub run, explicitly add an id if missing - id = "stub${index}" - } - assert id != null: "Each parameter set should have at least an 'id'" - - // Process params - def parValues = globalParams + tup[1] - // // Remove parameters which are null, if the default is also null - // parValues = parValues.collectEntries{paramName, paramValue -> - // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) - // if ( paramValue != null || parameterSettings.get("default", null) != null ) { - // [paramName, paramValue] - // } - // } - parValues = parValues.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" - - if (par == null) { - return [:] - } - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - - [ name, value ] - } - - [id, parValues] - } - - // Check if ids (first element of each list) is unique - _checkUniqueIds(processedParams) - return processedParams -} - -/** - * Parse nextflow parameters based on settings defined in a viash config - * and return a nextflow channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A nextflow Channel with events. Events are formatted as a tuple that contains - * first contains the ID of the event and as second element holds a parameter map. - * - * - */ -def channelFromParams(Map params, Map config) { - def processedParams = _paramsToParamSets(params, config) - return Channel.fromList(processedParams) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' -def checkUniqueIds(Map args) { - def stopOnError = args.stopOnError == null ? args.stopOnError : true - - def idChecker = new IDChecker() - - return filter { tup -> - if (!idChecker.observe(tup[0])) { - if (stopOnError) { - error "Duplicate id: ${tup[0]}" - } else { - log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" - return false - } - } - return true - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' -// This helper file will be deprecated soon -preprocessInputsDeprecationWarningPrinted = false - -def preprocessInputsDeprecationWarning() { - if (!preprocessInputsDeprecationWarningPrinted) { - preprocessInputsDeprecationWarningPrinted = true - System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") - } -} - -/** - * Generate a nextflow Workflow that allows processing a channel of - * Vdsl3 formatted events and apply a Viash config to them: - * - Gather default parameters from the Viash config and make - * sure that they are correctly formatted (see applyConfig method). - * - Format the input parameters (also using the applyConfig method). - * - Apply the default parameter to the input parameters. - * - Do some assertions: - * ~ Check if the event IDs in the channel are unique. - * - * The events in the channel are formatted as tuples, with the - * first element of the tuples being a unique id of the parameter set, - * and the second element containg the the parameters themselves. - * Optional extra elements of the tuples will be passed to the output as is. - * - * @param args A map that must contain a 'config' key that points - * to a parsed config (see readConfig()). Optionally, a - * 'key' key can be provided which can be used to create a unique - * name for the workflow process. - * - * @return A workflow that allows processing a channel of Vdsl3 formatted events - * and apply a Viash config to them. - */ -def preprocessInputs(Map args) { - preprocessInputsDeprecationWarning() - - def config = args.config - assert config instanceof Map : - "Error in preprocessInputs: config must be a map. " + - "Expected class: Map. Found: config.getClass() is ${config.getClass()}" - def key_ = args.key ?: config.name - - // Get different parameter types (used throughout this function) - def defaultArgs = config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - map { tup -> - def id = tup[0] - def data = tup[1] - def passthrough = tup.drop(2) - - def new_data = (defaultArgs + data).collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - - if (par != null) { - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - } - - [ name, value ] - } - - [ id, new_data ] + passthrough - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component config. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component config. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component config. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component config. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runComponents(Map args) { - log.warn("runComponents is deprecated, use runEach instead") - assert args.components: "runComponents should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runComponents" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def id_ = args.id - - workflow runComponentsWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def comp_config = comp_.config - - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_config) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - // def new_id = id_(tup[0], tup[1], comp_config) - def new_id = tup[0] - if (id_ instanceof String) { - new_id = id_ - } else if (id_ instanceof Closure) { - new_id = id_(new_id, tup[1], comp_config) - } - [new_id] + tup.drop(1) - } - : filter_ch - def data_ch = id_ch | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_config) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_config) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - post_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runComponentsWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component itself. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component itself. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component itself. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component itself. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runEach(Map args) { - assert args.components: "runEach should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runEach" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def runIf_ = args.runIf - def id_ = args.id - - assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." - - workflow runEachWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - def new_id = id_ - if (new_id instanceof Closure) { - new_id = new_id(tup[0], tup[1], comp_) - } - assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" - [new_id] + tup.drop(1) - } - : filter_ch - def chPassthrough = null - def chRun = null - if (runIf_) { - def idRunIfBranch = id_ch.branch{ tup -> - run: runIf_(tup[0], tup[1], comp_) - passthrough: true - } - chPassthrough = idRunIfBranch.passthrough - chRun = idRunIfBranch.run - } else { - chRun = id_ch - chPassthrough = Channel.empty() - } - def data_ch = chRun | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - def return_ch = post_ch - | concat(chPassthrough) - - return_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runEachWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' -/** - * Join sourceChannel to targetChannel - * - * This function joins the sourceChannel to the targetChannel. - * However, each id in the targetChannel must be present in the - * sourceChannel. If _meta.join_id exists in the targetChannel, that is - * used as an id instead. If the id doesn't match any id in the sourceChannel, - * an error is thrown. - */ - -def safeJoin(targetChannel, sourceChannel, key) { - def sourceIDs = new IDChecker() - - def sourceCheck = sourceChannel - | map { tup -> - sourceIDs.observe(tup[0]) - tup - } - def targetCheck = targetChannel - | map { tup -> - def id = tup[0] - - if (!sourceIDs.contains(id)) { - error ( - "Error in module '${key}' when merging output with original state.\n" + - " Reason: output with id '${id}' could not be joined with source channel.\n" + - " If the IDs in the output channel differ from the input channel,\n" + - " please set `tup[1]._meta.join_id to the original ID.\n" + - " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + - " Unexpected ID in the output channel: '${id}'.\n" + - " Example input event: [\"id\", [input: file(...)]],\n" + - " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" - ) - } - // TODO: add link to our documentation on how to fix this - - tup - } - - sourceCheck.cross(targetChannel) - | map{ left, right -> - right + left.drop(1) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' -def _processArgument(arg) { - arg.multiple = arg.multiple != null ? arg.multiple : false - arg.required = arg.required != null ? arg.required : false - arg.direction = arg.direction != null ? arg.direction : "input" - arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" - arg.plainName = arg.name.replaceAll("^-*", "") - - if (arg.type == "file") { - arg.must_exist = arg.must_exist != null ? arg.must_exist : true - arg.create_parent = arg.create_parent != null ? arg.create_parent : true - } - - // add default values to output files which haven't already got a default - if (arg.type == "file" && arg.direction == "output" && arg.default == null) { - def mult = arg.multiple ? "_*" : "" - def extSearch = "" - if (arg.default != null) { - extSearch = arg.default - } else if (arg.example != null) { - extSearch = arg.example - } - if (extSearch instanceof List) { - extSearch = extSearch[0] - } - def extSearchResult = extSearch.find("\\.[^\\.]+\$") - def ext = extSearchResult != null ? extSearchResult : "" - arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" - if (arg.multiple) { - arg.default = [arg.default] - } - } - - if (!arg.multiple) { - if (arg.default != null && arg.default instanceof List) { - arg.default = arg.default[0] - } - if (arg.example != null && arg.example instanceof List) { - arg.example = arg.example[0] - } - } - - if (arg.type == "boolean_true") { - arg.default = false - } - if (arg.type == "boolean_false") { - arg.default = true - } - - arg -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' -def addGlobalArguments(config) { - def localConfig = [ - "argument_groups": [ - [ - "name": "Nextflow input-output arguments", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "arguments" : [ - [ - 'name': '--publish_dir', - 'required': true, - 'type': 'string', - 'description': 'Path to an output directory.', - 'example': 'output/', - 'multiple': false - ], - [ - 'name': '--param_list', - 'required': false, - 'type': 'string', - 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. - | - |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. - |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. - |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. - |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. - | - |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), - 'example': 'my_params.yaml', - 'multiple': false, - 'hidden': true - ] - // TODO: allow multiple: true in param_list? - // TODO: allow to specify a --param_list_regex to filter the param_list? - // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? - ] - ] - ] - ] - - return processConfig(_mergeMap(config, localConfig)) -} - -def _mergeMap(Map lhs, Map rhs) { - return rhs.inject(lhs.clone()) { map, entry -> - if (map[entry.key] instanceof Map && entry.value instanceof Map) { - map[entry.key] = _mergeMap(map[entry.key], entry.value) - } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { - map[entry.key] += entry.value - } else { - map[entry.key] = entry.value - } - return map - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' -def _generateArgumentHelp(param) { - // alternatives are not supported - // def names = param.alternatives ::: List(param.name) - - def unnamedProps = [ - ["required parameter", param.required], - ["multiple values allowed", param.multiple], - ["output", param.direction.toLowerCase() == "output"], - ["file must exist", param.type == "file" && param.must_exist] - ].findAll{it[1]}.collect{it[0]} - - def dflt = null - if (param.default != null) { - if (param.default instanceof List) { - dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - dflt = param.default.toString() - } - } - def example = null - if (param.example != null) { - if (param.example instanceof List) { - example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - example = param.example.toString() - } - } - def min = param.min?.toString() - def max = param.max?.toString() - - def escapeChoice = { choice -> - def s1 = choice.replaceAll("\\n", "\\\\n") - def s2 = s1.replaceAll("\"", """\\\"""") - s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 - } - def choices = param.choices == null ? - null : - "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" - - def namedPropsStr = [ - ["type", ([param.type] + unnamedProps).join(", ")], - ["default", dflt], - ["example", example], - ["choices", choices], - ["min", min], - ["max", max] - ] - .findAll{it[1]} - .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} - .join("") - - def descStr = param.description == null ? - "" : - _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") - - "\n --" + param.plainName + - namedPropsStr + - descStr -} - -// Based on Helper.generateHelp() in Helper.scala -def _generateHelp(config) { - def fun = config - - // PART 1: NAME AND VERSION - def nameStr = fun.name + - (fun.version == null ? "" : " " + fun.version) - - // PART 2: DESCRIPTION - def descrStr = fun.description == null ? - "" : - "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") - - // PART 3: Usage - def usageStr = fun.usage == null ? - "" : - "\n\nUsage:\n" + fun.usage.trim() - - // PART 4: Options - def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> - def name = argGroup.name - def descriptionStr = argGroup.description == null ? - "" : - "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" - def arguments = argGroup.arguments.collect{arg -> - arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg - }.findAll{it != null} - def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} - - "\n\n$name:" + - descriptionStr + - argumentStrs.join("\n") - } - - // FINAL: combine - def out = nameStr + - descrStr + - usageStr + - argGroupStrs.join("") - - return out -} - -// based on Format._paragraphWrap -def _paragraphWrap(str, maxLength) { - def outLines = [] - str.split("\n").each{par -> - def words = par.split("\\s").toList() - - def word = null - def line = words.pop() - while(!words.isEmpty()) { - word = words.pop() - if (line.length() + word.length() + 1 <= maxLength) { - line = line + " " + word - } else { - outLines.add(line) - line = word - } - } - if (words.isEmpty()) { - outLines.add(line) - } - } - return outLines -} - -def helpMessage(config) { - if (params.containsKey("help") && params.help) { - def mergedConfig = addGlobalArguments(config) - def helpStr = _generateHelp(mergedConfig) - println(helpStr) - exit 0 - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' -def processConfig(config) { - // set defaults for arguments - config.arguments = - (config.arguments ?: []).collect{_processArgument(it)} - - // set defaults for argument_group arguments - config.argument_groups = - (config.argument_groups ?: []).collect{grp -> - grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} - grp - } - - // create combined arguments list - config.allArguments = - config.arguments + - config.argument_groups.collectMany{it.arguments} - - // add missing argument groups (based on Functionality::allArgumentGroups()) - def argGroups = config.argument_groups - if (argGroups.any{it.name.toLowerCase() == "arguments"}) { - argGroups = argGroups.collect{ grp -> - if (grp.name.toLowerCase() == "arguments") { - grp = grp + [ - arguments: grp.arguments + config.arguments - ] - } - grp - } - } else { - argGroups = argGroups + [ - name: "Arguments", - arguments: config.arguments - ] - } - config.allArgumentGroups = argGroups - - config -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' - -def readConfig(file) { - def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) - processConfig(config) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' -/** - * Resolve a path relative to the current file. - * - * @param str The path to resolve, as a String. - * @param parentPath The path to resolve relative to, as a Path. - * - * @return The path that may have been resovled, as a Path. - */ -def _resolveSiblingIfNotAbsolute(str, parentPath) { - if (str !instanceof String) { - return str - } - if (!_stringIsAbsolutePath(str)) { - return parentPath.resolveSibling(str) - } else { - return file(str, hidden: true) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' -/** - * Check whether a path as a string is absolute. - * - * In the past, we tried using `file(., relative: true).isAbsolute()`, - * but the 'relative' option was added in 22.10.0. - * - * @param path The path to check, as a String. - * - * @return Whether the path is absolute, as a boolean. - */ -def _stringIsAbsolutePath(path) { - def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ - - assert path instanceof String - return _resolve_URL_PROTOCOL.matcher(path).matches() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' -class CustomTraceObserver implements nextflow.trace.TraceObserver { - List traces - - CustomTraceObserver(List traces) { - this.traces = traces - } - - @Override - void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } - - @Override - void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } -} - -def collectTraces() { - def traces = Collections.synchronizedList([]) - - // add custom trace observer which stores traces in the traces object - session.observers.add(new CustomTraceObserver(traces)) - - traces -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' -/** - * Performs a deep clone of the given object. - * @param x an object - */ -def deepClone(x) { - iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' -def getPublishDir() { - return params.containsKey("publish_dir") ? params.publish_dir : - params.containsKey("publishDir") ? params.publishDir : - null -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' - -// Recurse upwards until we find a '.build.yaml' file -def _findBuildYamlFile(pathPossiblySymlink) { - def path = pathPossiblySymlink.toRealPath() - def child = path.resolve(".build.yaml") - if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { - return child - } else { - def parent = path.getParent() - if (parent == null) { - return null - } else { - return _findBuildYamlFile(parent) - } - } -} - -// get the root of the target folder -def getRootDir() { - def dir = _findBuildYamlFile(meta.resources_dir) - assert dir != null: "Could not find .build.yaml in the folder structure" - dir.getParent() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' -/** - * Recursively apply a function over the leaves of an object. - * @param obj The object to iterate over. - * @param fun The function to apply to each value. - * @return The object with the function applied to each value. - */ -def iterateMap(obj, fun) { - if (obj instanceof List && obj !instanceof String) { - return obj.collect{item -> - iterateMap(item, fun) - } - } else if (obj instanceof Map) { - return obj.collectEntries{key, item -> - [key.toString(), iterateMap(item, fun)] - } - } else { - return fun(obj) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' -/** - * A view for printing the event of each channel as a YAML blob. - * This is useful for debugging. - */ -def niceView() { - workflow niceViewWf { - take: input - main: - output = input - | view{toYamlBlob(it)} - emit: output - } - return niceViewWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' - -def readCsv(file_path) { - def output = [] - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - - // todo: allow escaped quotes in string - // todo: allow single quotes? - def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') - def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - - def br = java.nio.file.Files.newBufferedReader(inputFile) - - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field - } - } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) - } - } - - output -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' -def readJson(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parse(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' -def readJsonBlob(str) { - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parseText(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' -// Custom constructor to modify how certain objects are parsed from YAML -class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { - Path root - - class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { - public Object construct(org.yaml.snakeyaml.nodes.Node node) { - String filename = (String) constructScalar(node); - if (root != null) { - return root.resolve(filename); - } - return java.nio.file.Paths.get(filename); - } - } - - CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { - super(options) - this.root = root - // Handling !file tag and parse it back to a File type - this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) - } -} - -def readTaggedYaml(Path path) { - def options = new org.yaml.snakeyaml.LoaderOptions() - def constructor = new CustomConstructor(options, path.getParent()) - def yaml = new org.yaml.snakeyaml.Yaml(constructor) - return yaml.load(path.text) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' -def readYaml(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' -def readYamlBlob(str) { - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' -String toJsonBlob(data) { - return groovy.json.JsonOutput.toJson(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' -// Custom representer to modify how certain objects are represented in YAML -class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { - Path relativizer - - class RepresentPath implements org.yaml.snakeyaml.representer.Represent { - public String getFileName(Object obj) { - if (obj instanceof File) { - obj = ((File) obj).toPath(); - } - if (obj !instanceof Path) { - throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); - } - def path = (Path) obj; - - if (relativizer != null) { - return relativizer.relativize(path).toString() - } else { - return path.toString() - } - } - - public org.yaml.snakeyaml.nodes.Node representData(Object data) { - String filename = getFileName(data); - def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); - return representScalar(tag, filename); - } - } - CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { - super(options) - this.relativizer = relativizer - this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) - this.representers.put(Path, new RepresentPath()) - this.representers.put(File, new RepresentPath()) - } -} - -String toTaggedYamlBlob(data) { - return toRelativeTaggedYamlBlob(data, null) -} -String toRelativeTaggedYamlBlob(data, Path relativizer) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - def representer = new CustomRepresenter(options, relativizer) - def yaml = new org.yaml.snakeyaml.Yaml(representer, options) - return yaml.dump(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' -String toYamlBlob(data) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - options.setPrettyFlow(true) - def yaml = new org.yaml.snakeyaml.Yaml(options) - def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) - return yaml.dump(cleanData) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' -void writeJson(data, file) { - assert data: "writeJson: data should not be null" - assert file: "writeJson: file should not be null" - file.write(toJsonBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' -void writeYaml(data, file) { - assert data: "writeYaml: data should not be null" - assert file: "writeYaml: file should not be null" - file.write(toYamlBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' -def findStates(Map params, Map config) { - def auto_config = deepClone(config) - def auto_params = deepClone(params) - - auto_config = auto_config.clone() - // override arguments - auto_config.argument_groups = [] - auto_config.arguments = [ - [ - type: "string", - name: "--id", - description: "A dummy identifier", - required: false - ], - [ - type: "file", - name: "--input_states", - example: "/path/to/input/directory/**/state.yaml", - description: "Path to input directory containing the datasets to be integrated.", - required: true, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--filter", - example: "foo/.*/state.yaml", - description: "Regex to filter state files by path.", - required: false - ], - // to do: make this a yaml blob? - [ - type: "string", - name: "--rename_keys", - example: ["newKey1:oldKey1", "newKey2:oldKey2"], - description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", - required: false, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--settings", - example: '{"output_dataset": "dataset.h5ad", "k": 10}', - description: "Global arguments as a JSON glob to be passed to all components.", - required: false - ] - ] - if (!(auto_params.containsKey("id"))) { - auto_params["id"] = "auto" - } - - // run auto config through processConfig once more - auto_config = processConfig(auto_config) - - workflow findStatesWf { - helpMessage(auto_config) - - output_ch = - channelFromParams(auto_params, auto_config) - | flatMap { autoId, args -> - - def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] - - // look for state files in input dir - def stateFiles = args.input_states - - // filter state files by regex - if (args.filter) { - stateFiles = stateFiles.findAll{ stateFile -> - def stateFileStr = stateFile.toString() - def matcher = stateFileStr =~ args.filter - matcher.matches()} - } - - // read in states - def states = stateFiles.collect { stateFile -> - def state_ = readTaggedYaml(stateFile) - [state_.id, state_] - } - - // construct renameMap - if (args.rename_keys) { - def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(":") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" - split - } - - // rename keys in state, only let states through which have all keys - // also add global settings - states = states.collectMany{id, state -> - def newState = [:] - - for (key in renameMap.keySet()) { - def origKey = renameMap[key] - if (!(state.containsKey(origKey))) { - return [] - } - newState[key] = state[origKey] - } - - [[id, globalSettings + newState]] - } - } - - states - } - emit: - output_ch - } - - return findStatesWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' -def joinStates(Closure apply_) { - workflow joinStatesWf { - take: input_ch - main: - output_ch = input_ch - | toSortedList - | filter{ it.size() > 0 } - | map{ tups -> - def ids = tups.collect{it[0]} - def states = tups.collect{it[1]} - apply_(ids, states) - } - - emit: output_ch - } - return joinStatesWf -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' -def publishFiles(Map args) { - def key_ = args.get("key") - - assert key_ != null : "publishFiles: key must be specified" - - workflow publishFilesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] - - [id_, inputFiles_, outputFilenames_] - } - | publishFilesProc - emit: input_ch - } - return publishFilesWf -} - -process publishFilesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) - output: - tuple val(id), path{outputFiles} - script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } - """ - echo "Copying output files to destination folder" - ${copyCommands.join("\n ")} - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishFilesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishFilesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishFilesByConfig: key must be specified" - - workflow publishFilesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output OR multiple channels were emitted - // and the output was just not added to using the channel - // that is now being parsed - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[inputPath: [], outputFilename: []]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def inputPath = val instanceof File ? val.toPath() : val - [inputPath: inputPath, outputFilename: filename_ix] - } - def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] - } else { - def value_ = java.nio.file.Paths.get(filename) - def inputPath = value instanceof File ? value.toPath() : value - return [[inputPath: [inputPath], outputFilename: [filename]]] - } - } - - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} - - - [id_, inputPaths, outputFilenames] - } - | publishFilesProc - emit: input_ch - } - return publishFilesSimpleWf -} - - - - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' -def collectFiles(obj) { - if (obj instanceof java.io.File || obj instanceof Path) { - return [obj] - } else if (obj instanceof List && obj !instanceof String) { - return obj.collectMany{item -> - collectFiles(item) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectFiles(item) - } - } else { - return [] - } -} - -/** - * Recurse through a state and collect all input files and their target output filenames. - * @param obj The state to recurse through. - * @param prefix The prefix to prepend to the output filenames. - */ -def collectInputOutputPaths(obj, prefix) { - if (obj instanceof File || obj instanceof Path) { - def path = obj instanceof Path ? obj : obj.toPath() - def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" - def newFilename = prefix + ext - return [[obj, newFilename]] - } else if (obj instanceof List && obj !instanceof String) { - return obj.withIndex().collectMany{item, ix -> - collectInputOutputPaths(item, prefix + "_" + ix) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectInputOutputPaths(item, prefix + "." + key) - } - } else { - return [] - } -} - -def publishStates(Map args) { - def key_ = args.get("key") - def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) - - assert key_ != null : "publishStates: key must be specified" - - workflow publishStatesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - - def yamlFilename = yamlTemplate_ - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - - // TODO: do the pathnames in state_ match up with the outputFilenames_? - - // convert state to yaml blob - def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesWf -} -process publishStatesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), val(yamlBlob), val(yamlFile) - output: - tuple val(id), path{[yamlFile]} - script: - """ - mkdir -p "\$(dirname '${yamlFile}')" - echo "Storing state as yaml" - cat > '${yamlFile}' << HERE -${yamlBlob} -HERE - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishStatesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishStatesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishStatesByConfig: key must be specified" - - workflow publishStatesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - // TODO: allow overriding the state.yaml template - // TODO TODO: if auto.publish == "state", add output_state as an argument - def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' - def yamlFilename = yamlTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - - // the processed state is a list of [key, value] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - (key, value) are the tuples that will be saved to the state.yaml file - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[key: plainName_, value: value]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def value_ = java.nio.file.Paths.get(filename_ix) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - return value_ - } - return [["key": plainName_, "value": outputPerFile]] - } else { - def value_ = java.nio.file.Paths.get(filename) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - def inputPath = value instanceof File ? value.toPath() : value - return [["key": plainName_, value: value_]] - } - } - - - def updatedState_ = processedState.collectEntries{[it.key, it.value]} - - // convert state to yaml blob - def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesSimpleWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' -def setState(fun) { - assert fun instanceof Closure || fun instanceof Map || fun instanceof List : - "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" - - // if fun is a List, convert to map - if (fun instanceof List) { - // check whether fun is a list[string] - assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" - fun = fun.collectEntries{[it, it]} - } - - // if fun is a map, convert to closure - if (fun instanceof Map) { - // check whether fun is a map[string, string] - assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" - assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" - def funMap = fun.clone() - // turn the map into a closure to be used later on - fun = { id_, state_ -> - assert state_ instanceof Map : "Error in setState: the state is not a Map" - funMap.collectMany{newkey, origkey -> - if (state_.containsKey(origkey)) { - [[newkey, state_[origkey]]] - } else { - [] - } - }.collectEntries() - } - } - - map { tup -> - def id = tup[0] - def state = tup[1] - def unfilteredState = fun(id, state) - def newState = unfilteredState.findAll{key, val -> val != null} - [id, newState] + tup.drop(2) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' -// TODO: unit test processAuto -def processAuto(Map auto) { - // remove null values - auto = auto.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] - def unexpectedKeys = auto.keySet() - expectedKeys - assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" - - // check auto.simplifyInput - assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" - - // check auto.simplifyOutput - assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" - - // check auto.transcript - assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" - - // check auto.publish - assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" - - return auto.subMap(expectedKeys) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' -def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { - assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" - map.forEach { key, val -> - assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" - } - requiredKeys.forEach { requiredKey -> - assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" - } -} - -// TODO: unit test processDirectives -def processDirectives(Map drctv) { - // remove null values - drctv = drctv.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = [ - "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" - ] - def unexpectedKeys = drctv.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" - - /* DIRECTIVE accelerator - accepted examples: - - [ limit: 4, type: "nvidia-tesla-k80" ] - */ - if (drctv.containsKey("accelerator")) { - assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") - } - - /* DIRECTIVE afterScript - accepted examples: - - "source /cluster/bin/cleanup" - */ - if (drctv.containsKey("afterScript")) { - assert drctv["afterScript"] instanceof CharSequence - } - - /* DIRECTIVE beforeScript - accepted examples: - - "source /cluster/bin/setup" - */ - if (drctv.containsKey("beforeScript")) { - assert drctv["beforeScript"] instanceof CharSequence - } - - /* DIRECTIVE cache - accepted examples: - - true - - false - - "deep" - - "lenient" - */ - if (drctv.containsKey("cache")) { - assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean - if (drctv["cache"] instanceof CharSequence) { - assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" - } - } - - /* DIRECTIVE conda - accepted examples: - - "bwa=0.7.15" - - "bwa=0.7.15 fastqc=0.11.5" - - ["bwa=0.7.15", "fastqc=0.11.5"] - */ - if (drctv.containsKey("conda")) { - if (drctv["conda"] instanceof List) { - drctv["conda"] = drctv["conda"].join(" ") - } - assert drctv["conda"] instanceof CharSequence - } - - /* DIRECTIVE container - accepted examples: - - "foo/bar:tag" - - [ registry: "reg", image: "im", tag: "ta" ] - is transformed to "reg/im:ta" - - [ image: "im" ] - is transformed to "im:latest" - */ - if (drctv.containsKey("container")) { - assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence - if (drctv["container"] instanceof Map) { - def m = drctv["container"] - assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") - def part1 = - System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : - params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? - m.registry ? m.registry + "/" : - "" - def part2 = m.image - def part3 = m.tag ? ":" + m.tag : ":latest" - drctv["container"] = part1 + part2 + part3 - } - } - - /* DIRECTIVE containerOptions - accepted examples: - - "--foo bar" - - ["--foo bar", "-f b"] - */ - if (drctv.containsKey("containerOptions")) { - if (drctv["containerOptions"] instanceof List) { - drctv["containerOptions"] = drctv["containerOptions"].join(" ") - } - assert drctv["containerOptions"] instanceof CharSequence - } - - /* DIRECTIVE cpus - accepted examples: - - 1 - - 10 - */ - if (drctv.containsKey("cpus")) { - assert drctv["cpus"] instanceof Integer - } - - /* DIRECTIVE disk - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("disk")) { - assert drctv["disk"] instanceof CharSequence - // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE echo - accepted examples: - - true - - false - */ - if (drctv.containsKey("echo")) { - assert drctv["echo"] instanceof Boolean - } - - /* DIRECTIVE errorStrategy - accepted examples: - - "terminate" - - "finish" - */ - if (drctv.containsKey("errorStrategy")) { - assert drctv["errorStrategy"] instanceof CharSequence - assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" - } - - /* DIRECTIVE executor - accepted examples: - - "local" - - "sge" - */ - if (drctv.containsKey("executor")) { - assert drctv["executor"] instanceof CharSequence - assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" - } - - /* DIRECTIVE machineType - accepted examples: - - "n1-highmem-8" - */ - if (drctv.containsKey("machineType")) { - assert drctv["machineType"] instanceof CharSequence - } - - /* DIRECTIVE maxErrors - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxErrors")) { - assert drctv["maxErrors"] instanceof Integer - } - - /* DIRECTIVE maxForks - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxForks")) { - assert drctv["maxForks"] instanceof Integer - } - - /* DIRECTIVE maxRetries - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxRetries")) { - assert drctv["maxRetries"] instanceof Integer - } - - /* DIRECTIVE memory - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("memory")) { - assert drctv["memory"] instanceof CharSequence - // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE module - accepted examples: - - "ncbi-blast/2.2.27" - - "ncbi-blast/2.2.27:t_coffee/10.0" - - ["ncbi-blast/2.2.27", "t_coffee/10.0"] - */ - if (drctv.containsKey("module")) { - if (drctv["module"] instanceof List) { - drctv["module"] = drctv["module"].join(":") - } - assert drctv["module"] instanceof CharSequence - } - - /* DIRECTIVE penv - accepted examples: - - "smp" - */ - if (drctv.containsKey("penv")) { - assert drctv["penv"] instanceof CharSequence - } - - /* DIRECTIVE pod - accepted examples: - - [ label: "key", value: "val" ] - - [ annotation: "key", value: "val" ] - - [ env: "key", value: "val" ] - - [ [label: "l", value: "v"], [env: "e", value: "v"]] - */ - if (drctv.containsKey("pod")) { - if (drctv["pod"] instanceof Map) { - drctv["pod"] = [ drctv["pod"] ] - } - assert drctv["pod"] instanceof List - drctv["pod"].forEach { pod -> - assert pod instanceof Map - // TODO: should more checks be added? - // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod - // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? - } - } - - /* DIRECTIVE publishDir - accepted examples: - - [] - - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] - - "/path/to/dir" - is transformed to [[ path: "/path/to/dir" ]] - - [ path: "/path/to/dir", mode: "cache" ] - is transformed to [[ path: "/path/to/dir", mode: "cache" ]] - */ - // TODO: should we also look at params["publishDir"]? - if (drctv.containsKey("publishDir")) { - def pblsh = drctv["publishDir"] - - // check different options - assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence - - // turn into list if not already so - // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. - pblsh = pblsh instanceof List ? pblsh : [ pblsh ] - - // check elements of publishDir - pblsh = pblsh.collect{ elem -> - // turn into map if not already so - elem = elem instanceof CharSequence ? [ path: elem ] : elem - - // check types and keys - assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" - assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") - - // check elements in map - assert elem.containsKey("path") - assert elem["path"] instanceof CharSequence - if (elem.containsKey("mode")) { - assert elem["mode"] instanceof CharSequence - assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] - } - if (elem.containsKey("overwrite")) { - assert elem["overwrite"] instanceof Boolean - } - if (elem.containsKey("pattern")) { - assert elem["pattern"] instanceof CharSequence - } - if (elem.containsKey("saveAs")) { - assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" - } - if (elem.containsKey("enabled")) { - assert elem["enabled"] instanceof Boolean - } - - // return final result - elem - } - // store final directive - drctv["publishDir"] = pblsh - } - - /* DIRECTIVE queue - accepted examples: - - "long" - - "short,long" - - ["short", "long"] - */ - if (drctv.containsKey("queue")) { - if (drctv["queue"] instanceof List) { - drctv["queue"] = drctv["queue"].join(",") - } - assert drctv["queue"] instanceof CharSequence - } - - /* DIRECTIVE label - accepted examples: - - "big_mem" - - "big_cpu" - - ["big_mem", "big_cpu"] - */ - if (drctv.containsKey("label")) { - if (drctv["label"] instanceof CharSequence) { - drctv["label"] = [ drctv["label"] ] - } - assert drctv["label"] instanceof List - drctv["label"].forEach { label -> - assert label instanceof CharSequence - // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") - // ^ does not allow closures - } - } - - /* DIRECTIVE scratch - accepted examples: - - true - - "/path/to/scratch" - - '$MY_PATH_TO_SCRATCH' - - "ram-disk" - */ - if (drctv.containsKey("scratch")) { - assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence - } - - /* DIRECTIVE storeDir - accepted examples: - - "/path/to/storeDir" - */ - if (drctv.containsKey("storeDir")) { - assert drctv["storeDir"] instanceof CharSequence - } - - /* DIRECTIVE stageInMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageInMode")) { - assert drctv["stageInMode"] instanceof CharSequence - assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] - } - - /* DIRECTIVE stageOutMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageOutMode")) { - assert drctv["stageOutMode"] instanceof CharSequence - assert drctv["stageOutMode"] in ["copy", "move", "rsync"] - } - - /* DIRECTIVE tag - accepted examples: - - "foo" - - '$id' - */ - if (drctv.containsKey("tag")) { - assert drctv["tag"] instanceof CharSequence - } - - /* DIRECTIVE time - accepted examples: - - "1h" - - "2days" - - "1day 6hours 3minutes 30seconds" - */ - if (drctv.containsKey("time")) { - assert drctv["time"] instanceof CharSequence - // todo: validation regex? - } - - return drctv -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' -def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { - // override defaults with args - def workflowArgs = defaultWfArgs + args - - // check whether 'key' exists - assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" - - // if 'key' is a closure, apply it to the original key - if (workflowArgs["key"] instanceof Closure) { - workflowArgs["key"] = workflowArgs["key"](meta.config.name) - } - def key = workflowArgs["key"] - assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" - assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" - - // check for any unexpected keys - def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] - def unexpectedKeys = workflowArgs.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" - assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" - workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" - assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" - workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) - - // auto define publish, if so desired - if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + - // " Example: params.publish_dir = \"./output/\"" - def publishDir = getPublishDir() - - if (publishDir != null) { - workflowArgs.directives.publishDir = [[ - path: publishDir, - saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default - mode: "copy" - ]] - } - } - - // auto define transcript, if so desired - if (workflowArgs.auto.transcript == true) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + - // " Example: params.transcripts_dir = \"./transcripts/\"" - def transcriptsDir = - params.containsKey("transcripts_dir") ? params.transcripts_dir : - params.containsKey("transcriptsDir") ? params.transcriptsDir : - params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : - params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : - null - if (transcriptsDir != null) { - def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') - def transcriptsPublishDir = [ - path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", - saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", - mode: "copy" - ] - def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] - workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir - } - } - - // if this is a stubrun, remove certain directives? - if (workflow.stubRun) { - workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) - } - - for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { - assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" - } - } - - // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? - for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { - log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." - } - } - - // check fromState - workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) - - // check toState - workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) - - // return output - return workflowArgs -} - -def _processFromState(fromState, key_, config_) { - assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : - "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" - if (fromState == null) { - return null - } - - // if fromState is a List, convert to map - if (fromState instanceof List) { - // check whether fromstate is a list[string] - assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" - fromState = fromState.collectEntries{[it, it]} - } - - // if fromState is a map, convert to closure - if (fromState instanceof Map) { - // check whether fromstate is a map[string, string] - assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" - assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" - def fromStateMap = fromState.clone() - def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} - // turn the map into a closure to be used later on - fromState = { it -> - def state = it[1] - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def data = fromStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (state.containsKey(origkey)) { - [[newkey, state[origkey]]] - } else if (!requiredInputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") - } - }.collectEntries() - data - } - } - - return fromState -} - -def _processToState(toState, key_, config_) { - if (toState == null) { - toState = { tup -> tup[1] } - } - - // toState should be a closure, map[string, string], or list[string] - assert toState instanceof Closure || toState instanceof Map || toState instanceof List : - "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" - - // if toState is a List, convert to map - if (toState instanceof List) { - // check whether toState is a list[string] - assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" - toState = toState.collectEntries{[it, it]} - } - - // if toState is a map, convert to closure - if (toState instanceof Map) { - // check whether toState is a map[string, string] - assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" - assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" - def toStateMap = toState.clone() - def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} - // turn the map into a closure to be used later on - toState = { it -> - def output = it[1] - def state = it[2] - assert output instanceof Map : "Error in module '$key_': the output is not a Map" - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def extraEntries = toStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (output.containsKey(origkey)) { - [[newkey, output[origkey]]] - } else if (!requiredOutputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") - } - }.collectEntries() - state + extraEntries - } - } - - return toState -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' -def _debug(workflowArgs, debugKey) { - if (workflowArgs.debug) { - view { "process '${workflowArgs.key}' $debugKey tuple: $it" } - } else { - map { it } - } -} - -// depends on: innerWorkflowFactory -def workflowFactory(Map args, Map defaultWfArgs, Map meta) { - def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) - def key_ = workflowArgs["key"] - def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} - - workflow workflowInstance { - take: input_ - - main: - def chModified = input_ - | checkUniqueIds([:]) - | _debug(workflowArgs, "input") - | map { tuple -> - tuple = deepClone(tuple) - - if (workflowArgs.map) { - tuple = workflowArgs.map(tuple) - } - if (workflowArgs.mapId) { - tuple[0] = workflowArgs.mapId(tuple[0]) - } - if (workflowArgs.mapData) { - tuple[1] = workflowArgs.mapData(tuple[1]) - } - if (workflowArgs.mapPassthrough) { - tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) - } - - // check tuple - assert tuple instanceof List : - "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - assert tuple.size() >= 2 : - "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: tuple.size() == ${tuple.size()}" - - // check id field - if (tuple[0] instanceof GString) { - tuple[0] = tuple[0].toString() - } - assert tuple[0] instanceof CharSequence : - "Error in module '${key_}': first element of tuple in channel should be a String\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: ${tuple[0]}" - - // match file to input file - if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { - def inputFiles = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - - assert inputFiles.size() == 1 : - "Error in module '${key_}' id '${tuple[0]}'.\n" + - " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + - " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" - - tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() - } - - // check data field - assert tuple[1] instanceof Map : - "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // rename keys of data field in tuple - if (workflowArgs.renameKeys) { - assert workflowArgs.renameKeys instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" - assert tuple[1] instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // TODO: allow renameKeys to be a function? - workflowArgs.renameKeys.each { newKey, oldKey -> - assert newKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" - assert oldKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" - assert tuple[1].containsKey(oldKey) : - "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + - " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" - tuple[1].put(newKey, tuple[1][oldKey]) - } - tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) - } - tuple - } - - - def chRun = null - def chPassthrough = null - if (workflowArgs.runIf) { - def runIfBranch = chModified.branch{ tup -> - run: workflowArgs.runIf(tup[0], tup[1]) - passthrough: true - } - chRun = runIfBranch.run - chPassthrough = runIfBranch.passthrough - } else { - chRun = chModified - chPassthrough = Channel.empty() - } - - def chRunFiltered = workflowArgs.filter ? - chRun | filter{workflowArgs.filter(it)} : - chRun - - def chArgs = workflowArgs.fromState ? - chRunFiltered | map{ - def new_data = workflowArgs.fromState(it.take(2)) - [it[0], new_data] - } : - chRunFiltered | map {tup -> tup.take(2)} - - // fill in defaults - def chArgsWithDefaults = chArgs - | map { tuple -> - def id_ = tuple[0] - def data_ = tuple[1] - - // TODO: could move fromState to here - - // fetch default params from functionality - def defaultArgs = meta.config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - // fetch overrides in params - def paramArgs = meta.config.allArguments - .findAll { par -> - def argKey = key_ + "__" + par.plainName - params.containsKey(argKey) - } - .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } - - // fetch overrides in data - def dataArgs = meta.config.allArguments - .findAll { data_.containsKey(it.plainName) } - .collectEntries { [ it.plainName, data_[it.plainName] ] } - - // combine params - def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs - - // remove arguments with explicit null values - combinedArgs - .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} - - combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) - - [id_, combinedArgs] + tuple.drop(2) - } - - // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutputMulti = chArgsWithDefaults - | _debug(workflowArgs, "processed") - // run workflow - | innerWorkflowFactory(workflowArgs) - def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] - assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" - // Add a channel ID to the events, which designates the channel the event was emitted from as a running number - // This number is used to sort the events later when the events are gathered from across the channels. - def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> - def newChannel = channel - | map {tuple -> - assert tuple instanceof List : - "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - - def newEvent = [channelIndex] + tuple - return newEvent - } - return newChannel - } - // Put the events into 1 channel, cover case where there is only one channel is emitted - def chInitialOutput = chInitialOutputList.size() > 1 ? \ - chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ - chInitialOutputListWithIndexedEvents[0] - def chInitialOutputProcessed = chInitialOutput - | map { tuple -> - def channelId = tuple[0] - def id_ = tuple[1] - def output_ = tuple[2] - - // see if output map contains metadata - def meta_ = - output_ instanceof Map && output_.containsKey("_meta") ? - output_["_meta"] : - [:] - def join_id = meta_.join_id ?: id_ - - // remove metadata - output_ = output_.findAll{k, v -> k != "_meta"} - - // check value types - output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - - [join_id, channelId, id_, output_] - } - // | view{"chInitialOutput: ${it.take(3)}"} - - // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] - def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) - // input tuple format: [join_id, channel_id, id, output, prev_state, ...] - // output tuple format: [join_id, channel_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(2).take(3)) - tup.take(3) + [new_state] + tup.drop(5) - } - if (workflowArgs.auto.publish == "state") { - def chPublishFiles = chPublishWithPreviousState - // input tuple format: [join_id, channel_id, id, new_state, ...] - // output tuple format: [join_id, channel_id, id, new_state] - | map{ tup -> - tup.take(4) - } - - safeJoin(chPublishFiles, chArgsWithDefaults, key_) - // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(2).take(3) - } - | publishFilesByConfig(key: key_, config: meta.config) - } - // Join the state from the events that were emitted from different channels - def chJoined = chInitialOutputProcessed - | map {tuple -> - def join_id = tuple[0] - def channel_id = tuple[1] - def id = tuple[2] - def other = tuple.drop(3) - // Below, groupTuple is used to join the events. To make sure resuming a workflow - // keeps working, the output state must be deterministic. This means the state needs to be - // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', - // but hashing the state when it is large can be problematic in terms of performance. - // Therefore, a custom comparator function is provided. We add the channel ID to the - // states so that we can use the channel ID to sort the items. - def stateWithChannelID = [[channel_id] * other.size(), other].transpose() - // A comparator that is provided to groupTuple's 'sort' argument is applied - // to all elements of the event tuple (that is not the 'id'). The comparator - // closure that is used below expects the input to be List. So the join_id and - // channel_id must also be wrapped in a list. - [[join_id], [channel_id], id] + stateWithChannelID - } - | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) - | map {join_ids, _, id, statesWithChannelID -> - // Remove the channel IDs from the states - def states = statesWithChannelID.collect{it[1]} - def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} - assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." - def newJoinIdUnique = newJoinId[0] - - // Merge the states from the different channels - def newState = states.inject([:]){ old_state, state_to_add -> - return old_state + state_to_add.collectEntries{k, v -> - if (!multipleArgs.contains(k)) { - // if the key is not a multiple argument, we expect only one value - if (old_state.containsKey(k)) { - assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." - } - [k, v] - } else { - // if the key is a multiple argument, append the different values into one list - def prevValue = old_state.getOrDefault(k, []) - def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] - [k, prevValueAsList + v] - } - } - } - - _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) - - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { - newState = newState.values()[0] - } - - return [newJoinIdUnique, id, newState] - } - - // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chJoined, chRunFiltered, key_) - // input tuple format: [join_id, id, output, prev_state, ...] - // output tuple format: [join_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(1).take(3)) - tup.take(2) + [new_state] + tup.drop(4) - } - - if (workflowArgs.auto.publish == "state") { - def chPublishStates = chNewState - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [join_id, id, new_state] - | map{ tup -> - tup.take(3) - } - - safeJoin(chPublishStates, chArgsWithDefaults, key_) - // input tuple format: [join_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(1).take(3) - } - | publishStatesByConfig(key: key_, config: meta.config) - } - chReturn = chNewState - | map { tup -> - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [id, new_state, ...] - tup.drop(1) - } - | _debug(workflowArgs, "output") - | concat(chPassthrough) - - emit: chReturn - } - - def wf = workflowInstance.cloneWithName(key_) - - // add factory function - wf.metaClass.run = { runArgs -> - workflowFactory(runArgs, workflowArgs, meta) - } - // add config to module for later introspection - wf.metaClass.config = meta.config - - return wf -} - -nextflow.enable.dsl=2 - -// START COMPONENT-SPECIFIC CODE - -// create meta object -meta = [ - "resources_dir": moduleDir.toRealPath().normalize(), - "config": processConfig(readJsonBlob('''{ - "name" : "calculate_qc_metrics", - "namespace" : "qc", - "version" : "disable-scrublet_build", - "authors" : [ - { - "name" : "Dries Schaumont", - "roles" : [ - "author" - ], - "info" : { - "role" : "Core Team Member", - "links" : { - "email" : "dries@data-intuitive.com", - "github" : "DriesSchaumont", - "orcid" : "0000-0002-4389-0440", - "linkedin" : "dries-schaumont" - }, - "organizations" : [ - { - "name" : "Data Intuitive", - "href" : "https://www.data-intuitive.com", - "role" : "Data Scientist" - } - ] - } - } - ], - "argument_groups" : [ - { - "name" : "Inputs", - "arguments" : [ - { - "type" : "file", - "name" : "--input", - "description" : "Input h5mu file", - "example" : [ - "input.h5mu" - ], - "must_exist" : true, - "create_parent" : true, - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--modality", - "description" : "Which modality from the input MuData file to process. \n", - "default" : [ - "rna" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--layer", - "description" : "Layer from modality to use as input data. If not provided the .X attribute is used.\n", - "example" : [ - "raw_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Metrics added to .obs", - "arguments" : [ - { - "type" : "string", - "name" : "--var_qc_metrics", - "description" : "Keys to select a boolean (containing only True or False) column from .var.\nFor each cell, calculate the proportion of total values for genes which are labeled 'True', \ncompared to the total sum of the values for all genes.\n", - "example" : [ - "ercc,highly_variable,mitochondrial" - ], - "required" : false, - "direction" : "input", - "multiple" : true, - "multiple_sep" : ";" - }, - { - "type" : "boolean", - "name" : "--var_qc_metrics_fill_na_value", - "description" : "Fill any 'NA' values found in the columns specified with --var_qc_metrics to 'True' or 'False'.\nas False.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "integer", - "name" : "--top_n_vars", - "description" : "Number of top vars to be used to calculate cumulative proportions.\nIf not specified, proportions are not calculated. `--top_n_vars 20;50` finds\ncumulative proportion to the 20th and 50th most expressed vars.\n", - "required" : false, - "direction" : "input", - "multiple" : true, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_obs_num_nonzero_vars", - "description" : "Name of column in .obs describing, for each observation, the number of stored values\n(including explicit zeroes). In other words, the name of the column that counts\nfor each row the number of columns that contain data.\n", - "default" : [ - "num_nonzero_vars" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_obs_total_counts_vars", - "description" : "Name of the column for .obs describing, for each observation (row),\nthe sum of the stored values in the columns.\n", - "default" : [ - "total_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Metrics added to .var", - "arguments" : [ - { - "type" : "string", - "name" : "--output_var_num_nonzero_obs", - "description" : "Name of column describing, for each feature, the number of stored values\n(including explicit zeroes). In other words, the name of the column that counts\nfor each column the number of rows that contain data.\n", - "default" : [ - "num_nonzero_obs" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_total_counts_obs", - "description" : "Name of the column in .var describing, for each feature (column),\nthe sum of the stored values in the rows.\n", - "default" : [ - "total_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_obs_mean", - "description" : "Name of the column in .obs providing the mean of the values in each row.\n", - "default" : [ - "obs_mean" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_pct_dropout", - "description" : "Name of the column in .obs providing for each feature the percentage of\nobservations the feature does not appear on (i.e. is missing). Same as `--num_nonzero_obs`\nbut percentage based.\n", - "default" : [ - "pct_dropout" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Outputs", - "arguments" : [ - { - "type" : "file", - "name" : "--output", - "description" : "Output h5mu file.", - "example" : [ - "output.h5mu" - ], - "must_exist" : true, - "create_parent" : true, - "required" : false, - "direction" : "output", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_compression", - "description" : "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", - "example" : [ - "gzip" - ], - "required" : false, - "choices" : [ - "gzip", - "lzf" - ], - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - } - ], - "resources" : [ - { - "type" : "python_script", - "path" : "script.py", - "is_executable" : true - }, - { - "type" : "file", - "path" : "/src/utils/setup_logger.py" - }, - { - "type" : "file", - "path" : "/src/utils/compress_h5mu.py" - }, - { - "type" : "file", - "path" : "/src/workflows/utils/labels.config", - "dest" : "nextflow_labels.config" - } - ], - "description" : "Add basic quality control metrics to an .h5mu file.\n\nThe metrics are comparable to what scanpy.pp.calculate_qc_metrics output,\nalthough they have slightly different names:\n\nVar metrics (name in this component -> name in scanpy):\n - pct_dropout -> pct_dropout_by_{expr_type}\n - num_nonzero_obs -> n_cells_by_{expr_type}\n - obs_mean -> mean_{expr_type}\n - total_counts -> total_{expr_type}\n\n Obs metrics:\n - num_nonzero_vars -> n_genes_by_{expr_type}\n - pct_{var_qc_metrics} -> pct_{expr_type}_{qc_var}\n - total_counts_{var_qc_metrics} -> total_{expr_type}_{qc_var}\n - pct_of_counts_in_top_{top_n_vars}_vars -> pct_{expr_type}_in_top_{n}_{var_type}\n - total_counts -> total_{expr_type}\n \n", - "test_resources" : [ - { - "type" : "python_script", - "path" : "test.py", - "is_executable" : true - }, - { - "type" : "file", - "path" : "/resources_test/pbmc_1k_protein_v3/pbmc_1k_protein_v3_filtered_feature_bc_matrix.h5mu" - } - ], - "status" : "enabled", - "scope" : { - "image" : "public", - "target" : "public" - }, - "license" : "MIT", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io" - }, - "runners" : [ - { - "type" : "executable", - "id" : "executable", - "docker_setup_strategy" : "ifneedbepullelsecachedbuild" - }, - { - "type" : "nextflow", - "id" : "nextflow", - "directives" : { - "label" : [ - "singlecpu", - "midmem" - ], - "tag" : "$id" - }, - "auto" : { - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false - }, - "config" : { - "labels" : { - "mem1gb" : "memory = 1000000000.B", - "mem2gb" : "memory = 2000000000.B", - "mem5gb" : "memory = 5000000000.B", - "mem10gb" : "memory = 10000000000.B", - "mem20gb" : "memory = 20000000000.B", - "mem50gb" : "memory = 50000000000.B", - "mem100gb" : "memory = 100000000000.B", - "mem200gb" : "memory = 200000000000.B", - "mem500gb" : "memory = 500000000000.B", - "mem1tb" : "memory = 1000000000000.B", - "mem2tb" : "memory = 2000000000000.B", - "mem5tb" : "memory = 5000000000000.B", - "mem10tb" : "memory = 10000000000000.B", - "mem20tb" : "memory = 20000000000000.B", - "mem50tb" : "memory = 50000000000000.B", - "mem100tb" : "memory = 100000000000000.B", - "mem200tb" : "memory = 200000000000000.B", - "mem500tb" : "memory = 500000000000000.B", - "mem1gib" : "memory = 1073741824.B", - "mem2gib" : "memory = 2147483648.B", - "mem4gib" : "memory = 4294967296.B", - "mem8gib" : "memory = 8589934592.B", - "mem16gib" : "memory = 17179869184.B", - "mem32gib" : "memory = 34359738368.B", - "mem64gib" : "memory = 68719476736.B", - "mem128gib" : "memory = 137438953472.B", - "mem256gib" : "memory = 274877906944.B", - "mem512gib" : "memory = 549755813888.B", - "mem1tib" : "memory = 1099511627776.B", - "mem2tib" : "memory = 2199023255552.B", - "mem4tib" : "memory = 4398046511104.B", - "mem8tib" : "memory = 8796093022208.B", - "mem16tib" : "memory = 17592186044416.B", - "mem32tib" : "memory = 35184372088832.B", - "mem64tib" : "memory = 70368744177664.B", - "mem128tib" : "memory = 140737488355328.B", - "mem256tib" : "memory = 281474976710656.B", - "mem512tib" : "memory = 562949953421312.B", - "cpu1" : "cpus = 1", - "cpu2" : "cpus = 2", - "cpu5" : "cpus = 5", - "cpu10" : "cpus = 10", - "cpu20" : "cpus = 20", - "cpu50" : "cpus = 50", - "cpu100" : "cpus = 100", - "cpu200" : "cpus = 200", - "cpu500" : "cpus = 500", - "cpu1000" : "cpus = 1000" - }, - "script" : [ - "includeConfig(\\"nextflow_labels.config\\")" - ] - }, - "debug" : false, - "container" : "docker" - } - ], - "engines" : [ - { - "type" : "docker", - "id" : "docker", - "image" : "python:3.11-slim", - "target_tag" : "disable-scrublet_build", - "namespace_separator" : "/", - "setup" : [ - { - "type" : "apt", - "packages" : [ - "procps" - ], - "interactive" : false - }, - { - "type" : "python", - "user" : false, - "packages" : [ - "anndata~=0.11.1", - "mudata~=0.3.1", - "scipy" - ], - "script" : [ - "exec(\\"try:\\\\n import awkward\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: exit(1)\\")" - ], - "upgrade" : true - } - ], - "test_setup" : [ - { - "type" : "apt", - "packages" : [ - "git" - ], - "interactive" : false - }, - { - "type" : "python", - "user" : false, - "packages" : [ - "viashpy==0.8.0" - ], - "github" : [ - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" - ], - "upgrade" : true - }, - { - "type" : "python", - "user" : false, - "packages" : [ - "scanpy" - ], - "upgrade" : true - } - ] - } - ], - "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/qc/calculate_qc_metrics/config.vsh.yaml", - "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/qc/calculate_qc_metrics", - "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", - "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" - }, - "package_config" : { - "name" : "openpipeline", - "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", - "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", - "info" : { - "test_resources" : [ - { - "type" : "s3", - "path" : "s3://openpipelines-data", - "dest" : "resources_test" - } - ] - }, - "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", - "config_mods" : [ - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" - ], - "keywords" : [ - "single-cell", - "multimodal" - ], - "license" : "MIT", - "organization" : "openpipelines-bio", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io", - "homepage" : "https://openpipelines.bio", - "documentation" : "https://openpipelines.bio/fundamentals", - "issue_tracker" : "https://github.com/openpipelines-bio/openpipeline/issues" - } - } -}''')) -] - -// resolve dependencies dependencies (if any) - - -// inner workflow -// inner workflow hook -def innerWorkflowFactory(args) { - def rawScript = '''set -e -tempscript=".viash_script.py" -cat > "$tempscript" << VIASHMAIN -import sys -from mudata import read_h5ad -from scipy.sparse import issparse, csr_array -import numpy as np - -## VIASH START -# The following code has been auto-generated by Viash. -par = { - 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'layer': $( if [ ! -z ${VIASH_PAR_LAYER+x} ]; then echo "r'${VIASH_PAR_LAYER//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'var_qc_metrics': $( if [ ! -z ${VIASH_PAR_VAR_QC_METRICS+x} ]; then echo "r'${VIASH_PAR_VAR_QC_METRICS//\\'/\\'\\"\\'\\"r\\'}'.split(';')"; else echo None; fi ), - 'var_qc_metrics_fill_na_value': $( if [ ! -z ${VIASH_PAR_VAR_QC_METRICS_FILL_NA_VALUE+x} ]; then echo "r'${VIASH_PAR_VAR_QC_METRICS_FILL_NA_VALUE//\\'/\\'\\"\\'\\"r\\'}'.lower() == 'true'"; else echo None; fi ), - 'top_n_vars': $( if [ ! -z ${VIASH_PAR_TOP_N_VARS+x} ]; then echo "list(map(int, r'${VIASH_PAR_TOP_N_VARS//\\'/\\'\\"\\'\\"r\\'}'.split(';')))"; else echo None; fi ), - 'output_obs_num_nonzero_vars': $( if [ ! -z ${VIASH_PAR_OUTPUT_OBS_NUM_NONZERO_VARS+x} ]; then echo "r'${VIASH_PAR_OUTPUT_OBS_NUM_NONZERO_VARS//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_obs_total_counts_vars': $( if [ ! -z ${VIASH_PAR_OUTPUT_OBS_TOTAL_COUNTS_VARS+x} ]; then echo "r'${VIASH_PAR_OUTPUT_OBS_TOTAL_COUNTS_VARS//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_var_num_nonzero_obs': $( if [ ! -z ${VIASH_PAR_OUTPUT_VAR_NUM_NONZERO_OBS+x} ]; then echo "r'${VIASH_PAR_OUTPUT_VAR_NUM_NONZERO_OBS//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_var_total_counts_obs': $( if [ ! -z ${VIASH_PAR_OUTPUT_VAR_TOTAL_COUNTS_OBS+x} ]; then echo "r'${VIASH_PAR_OUTPUT_VAR_TOTAL_COUNTS_OBS//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_var_obs_mean': $( if [ ! -z ${VIASH_PAR_OUTPUT_VAR_OBS_MEAN+x} ]; then echo "r'${VIASH_PAR_OUTPUT_VAR_OBS_MEAN//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_var_pct_dropout': $( if [ ! -z ${VIASH_PAR_OUTPUT_VAR_PCT_DROPOUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT_VAR_PCT_DROPOUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'output_compression': $( if [ ! -z ${VIASH_PAR_OUTPUT_COMPRESSION+x} ]; then echo "r'${VIASH_PAR_OUTPUT_COMPRESSION//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ) -} -meta = { - 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), - 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), - 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) -} -dep = { - -} - -## VIASH END - -sys.path.append(meta["resources_dir"]) -from setup_logger import setup_logger -from compress_h5mu import write_h5ad_to_h5mu_with_compression - -logger = setup_logger() - - -def count_nonzero(layer, axis): - """ - This method is the functional equivalent of the old .getnnz function from scirpy, - but that function was deprecated. So we use the nonzero function to mimic the old - behavior. - """ - axis ^= 1 - nonzero_counts = dict(zip(*np.unique(layer.nonzero()[axis], return_counts=True))) - nonzero_per_axis_item = { - row_index: nonzero_counts.get(row_index, 0) - for row_index in range(layer.shape[axis]) - } - return np.array(list(nonzero_per_axis_item.values()), dtype="int64") - - -def main(): - modality_data = read_h5ad(par["input"], mod=par["modality"]) - var = modality_data.var - layer = modality_data.X if not par["layer"] else modality_data.layers[par["layer"]] - if not issparse(layer): - raise NotImplementedError("Expected layer to be in sparse format.") - layer = csr_array(layer) - layer.eliminate_zeros() - - var_columns_to_add = {} - - # var statistics - if par["output_var_obs_mean"]: - obs_mean = layer.mean(axis=0) - var_columns_to_add[par["output_var_obs_mean"]] = obs_mean - if par["output_var_total_counts_obs"]: - # from the np.sum documentation: - # Especially when summing a large number of lower precision floating point numbers, - # such as float32, numerical errors can become significant. In such cases it can - # be advisable to use dtype="float64" to use a higher precision for the output. - layer_with_type = layer - if np.issubdtype(layer.dtype, np.floating) and np.can_cast( - layer.dtype, np.float64, casting="safe" - ): - # 'safe' casting makes sure not to cast np.float128 or anything else to a lower precision dtype - layer_with_type = layer.astype(np.float64) - total_counts_obs = np.ravel(layer_with_type.sum(axis=0)) - var_columns_to_add[par["output_var_total_counts_obs"]] = total_counts_obs - - num_nonzero_obs = count_nonzero(layer, axis=0) - if par["output_var_num_nonzero_obs"]: - var_columns_to_add[par["output_var_num_nonzero_obs"]] = num_nonzero_obs - if par["output_var_pct_dropout"]: - var_columns_to_add[par["output_var_pct_dropout"]] = ( - 1 - num_nonzero_obs / layer.shape[0] - ) * 100 - - modality_data.var = modality_data.var.assign(**var_columns_to_add) - - # obs statistics - obs_columns_to_add = {} - total_counts_var = np.ravel(layer.sum(axis=1)) - - if par["output_obs_num_nonzero_vars"]: - num_nonzero_vars = count_nonzero(layer, axis=1) - obs_columns_to_add[par["output_obs_num_nonzero_vars"]] = num_nonzero_vars - - if par["output_obs_total_counts_vars"]: - obs_columns_to_add[par["output_obs_total_counts_vars"]] = total_counts_var - - top_metrics = {} - if par["top_n_vars"]: - par["top_n_vars"] = sorted(par["top_n_vars"]) - distributions = get_top_from_csr_matrix(layer, par["top_n_vars"]) - top_metrics = { - distribution_size: distribution * 100 - for distribution_size, distribution in zip( - par["top_n_vars"], distributions.T - ) - } - obs_columns_to_add |= { - f"pct_of_counts_in_top_{n_top}_vars": col - for n_top, col in top_metrics.items() - } - - if par["var_qc_metrics"]: - print(f"qc_metrics: {par['var_qc_metrics']}") - for qc_metric in par["var_qc_metrics"]: - if qc_metric not in var: - raise ValueError( - f"Value for --var_qc_metrics, '{qc_metric}' " - f"not found in .var for modality {par['modality']}" - ) - qc_column = var[qc_metric] - if qc_column.isna().any(): - if par["var_qc_metrics_fill_na_value"] is None: - raise ValueError( - f"The .var column '{qc_metric}', selected by '--var_qc_metrics', contains NA values. " - "It is ambiguous whether or not to include these values in the static calulation. " - "You can explicitly map the NA values to 'False' or 'True using '--var_qc_metrics_fill_na_value'" - ) - else: - qc_column = qc_column.fillna( - par["var_qc_metrics_fill_na_value"], inplace=False - ) - qc_column = qc_column.to_list() - if set(np.unique(qc_column)) - {True, False}: - raise ValueError( - f"Column {qc_metric} in .var for modality {par['modality']} " - f"must only contain boolean values" - ) - total_counts_qc_metric = np.ravel(layer[:, qc_column].sum(axis=1)) - obs_columns_to_add |= { - f"total_counts_{qc_metric}": total_counts_qc_metric, - f"pct_{qc_metric}": total_counts_qc_metric / total_counts_var * 100, - } - - modality_data.obs = modality_data.obs.assign(**obs_columns_to_add) - write_h5ad_to_h5mu_with_compression( - par["output"], - par["input"], - par["modality"], - modality_data, - par["output_compression"], - ) - - -def get_top_from_csr_matrix(array, top_n_genes): - # csr matrices stores a 3D matrix in a format such that data for individual cells - # are stored in 1 array. Another array (indptr) here stores the ranges of indices - # to select from the data-array (.e.g. data[indptr[0]:indptr[1]] for row 0) for each row. - # Another array 'indices' maps each element of data to a column - # (data and indices arrays have the same length) - top_n_genes = np.array(top_n_genes).astype(np.int64) - assert np.all(top_n_genes[:-1] <= top_n_genes[1:]), "top_n_genes must be sorted" - row_indices, data = array.indptr, array.data - number_of_rows, max_genes_to_parse = row_indices.size - 1, top_n_genes[-1] - top_data = np.zeros((number_of_rows, max_genes_to_parse), dtype=data.dtype) - # Loop over each row to create a dense matrix without the 0 counts, - # but not for the whole matrix, only store the genes up until - # the largest number of top n genes. - for row_number in range(number_of_rows): - row_start_index, row_end_index = ( - row_indices[row_number], - row_indices[row_number + 1], - ) - row_data = data[row_start_index:row_end_index] # all non-zero counts for an row - try: - # There are less genes with counts in the row than the - # maximum number of genes we would like to select - # all these genes are in the top genes, just store them - top_data[row_number, : row_end_index - row_start_index] = row_data - except ValueError: - # Store the counts for the top genes - top_data[row_number, :] = np.partition(row_data, -max_genes_to_parse)[ - -max_genes_to_parse: - ] - - # Partition works from smallest to largest, but we want largest - # so do smallest to largest first (but with reversed indices) - top_data = np.partition(top_data, max_genes_to_parse - top_n_genes) - # And then switch the order around - top_data = np.flip(top_data, axis=1) - - cumulative = top_data.cumsum(axis=1, dtype=np.float64)[:, top_n_genes - 1] - return cumulative / np.expand_dims(array.sum(axis=1), 1) - - -if __name__ == "__main__": - main() -VIASHMAIN -python -B "$tempscript" -''' - - return vdsl3WorkflowFactory(args, meta, rawScript) -} - - - -/** - * Generate a workflow for VDSL3 modules. - * - * This function is called by the workflowFactory() function. - * - * Input channel: [id, input_map] - * Output channel: [id, output_map] - * - * Internally, this workflow will convert the input channel - * to a format which the Nextflow module will be able to handle. - */ -def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { - def key = args["key"] - def processObj = null - - workflow processWf { - take: input_ - main: - - if (processObj == null) { - processObj = _vdsl3ProcessFactory(args, meta, rawScript) - } - - output_ = input_ - | map { tuple -> - def id = tuple[0] - def data_ = tuple[1] - - if (workflow.stubRun) { - // add id if missing - data_ = [id: 'stub'] + data_ - } - - // process input files separately - def inputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - .collect { par -> - def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] - def inputFiles = [] - if (val == null) { - inputFiles = [] - } else if (val instanceof List) { - inputFiles = val - } else if (val instanceof Path) { - inputFiles = [ val ] - } else { - inputFiles = [] - } - if (!workflow.stubRun) { - // throw error when an input file doesn't exist - inputFiles.each{ file -> - assert file.exists() : - "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + - " Required input file does not exist.\n" + - " Path: '$file'.\n" + - " Expected input file to exist" - } - } - inputFiles - } - - // remove input files - def argsExclInputFiles = meta.config.allArguments - .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } - .collectEntries { par -> - def parName = par.plainName - def val = data_[parName] - if (par.multiple && val instanceof Collection) { - val = val.join(par.multiple_sep) - } - if (par.direction == "output" && par.type == "file") { - val = val - .replaceAll('\\$id', id) - .replaceAll('\\$\\{id\\}', id) - .replaceAll('\\$key', key) - .replaceAll('\\$\\{key\\}', key) - } - [parName, val] - } - - [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] - } - | processObj - | map { output -> - def outputFiles = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .indexed() - .collectEntries{ index, par -> - def out = output[index + 1] - // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) - if (!out instanceof List || out.size() <= 1) { - if (par.multiple) { - out = [] - } else { - assert !par.required : - "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + - " Required output file is missing" - out = null - } - } else if (out.size() == 2 && !par.multiple) { - out = out[1] - } else { - out = out.drop(1) - } - [ par.plainName, out ] - } - - // drop null outputs - outputFiles.removeAll{it.value == null} - - [ output[0], outputFiles ] - } - emit: output_ - } - - return processWf -} - -// depends on: session? -def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { - // autodetect process key - def wfKey = workflowArgs["key"] - def procKeyPrefix = "${wfKey}_process" - def scriptMeta = nextflow.script.ScriptMeta.current() - def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} - def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} - def newNumber = (numbers + [-1]).max() + 1 - - def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" - - if (newNumber > 0) { - log.warn "Key for module '${wfKey}' is duplicated.\n", - "If you run a component multiple times in the same workflow,\n" + - "it's recommended you set a unique key for every call,\n" + - "for example: ${wfKey}.run(key: \"foo\")." - } - - // subset directives and convert to list of tuples - def drctv = workflowArgs.directives - - // TODO: unit test the two commands below - // convert publish array into tags - def valueToStr = { val -> - // ignore closures - if (val instanceof CharSequence) { - if (!val.matches('^[{].*[}]$')) { - '"' + val + '"' - } else { - val - } - } else if (val instanceof List) { - "[" + val.collect{valueToStr(it)}.join(", ") + "]" - } else if (val instanceof Map) { - "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" - } else { - val.inspect() - } - } - - // multiple entries allowed: label, publishdir - def drctvStrs = drctv.collect { key, value -> - if (key in ["label", "publishDir"]) { - value.collect{ val -> - if (val instanceof Map) { - "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") - } else if (val == null) { - "" - } else { - "\n$key " + valueToStr(val) - } - }.join() - } else if (value instanceof Map) { - "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") - } else { - "\n$key " + valueToStr(value) - } - }.join() - - def inputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } - .join() - - def outputPaths = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .collect { par -> - // insert dummy into every output (see nextflow-io/nextflow#2678) - if (!par.multiple) { - ', path{[".exitcode", args.' + par.plainName + ']}' - } else { - ', path{[".exitcode"] + args.' + par.plainName + '}' - } - } - .join() - - // TODO: move this functionality somewhere else? - if (workflowArgs.auto.transcript) { - outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' - } else { - outputPaths = outputPaths + ', path{[".exitcode"]}' - } - - // create dirs for output files (based on BashWrapper.createParentFiles) - def createParentStr = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" && it.create_parent } - .collect { par -> - def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" - "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" - } - .join("\n") - - // construct inputFileExports - def inputFileExports = meta.config.allArguments - .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } - .collect { par -> - def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" - "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" - } - - // NOTE: if using docker, use /tmp instead of tmpDir! - def tmpDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('VIASH_TMPDIR') ?: - System.getenv('VIASH_TEMPDIR') ?: - System.getenv('VIASH_TMP') ?: - System.getenv('TEMP') ?: - System.getenv('TMPDIR') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMP') ?: - '/tmp' - ).toAbsolutePath() - - // construct stub - def stub = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "output" } - .collect { par -> - "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" - } - .join("\n") - - // escape script - def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') - - // publishdir assert - def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? - """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : - "" - - // generate process string - def procStr = - """nextflow.enable.dsl=2 - | - |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } - |process $procKey {$drctvStrs - |input: - | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") - |output: - | tuple val("\$id")$outputPaths, optional: true - |stub: - |\"\"\" - |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } - |$stub - |\"\"\" - |script:$assertStr - |def parInject = args - | .findAll{key, value -> value != null} - | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} - | .join("\\n") - |\"\"\" - |# meta exports - |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" - |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" - |export VIASH_META_NAME="${meta.config.name}" - |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" - |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" - |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } - |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } - |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then - | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) - | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) - | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) - | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) - | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) - | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) - | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) - | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) - | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) - | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) - |fi - | - |# meta synonyms - |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" - |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" - | - |# create output dirs if need be - |function mkdir_parent { - | for file in "\\\$@"; do - | mkdir -p "\\\$(dirname "\\\$file")" - | done - |} - |$createParentStr - | - |# argument exports${inputFileExports.join()} - |\$parInject - | - |# process script - |${escapedScript} - |\"\"\" - |} - |""".stripMargin() - - // TODO: print on debug - // if (workflowArgs.debug == true) { - // println("######################\n$procStr\n######################") - // } - - // write process to temp file - def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") - addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } - tempFile.text = procStr - - // create process from temp file - def binding = new nextflow.script.ScriptBinding([:]) - def session = nextflow.Nextflow.getSession() - def parser = _getScriptLoader(session) - .setModule(true) - .setBinding(binding) - def moduleScript = parser.runScript(tempFile) - .getScript() - - // register module in meta - def module = new nextflow.script.IncludeDef.Module(name: procKey) - scriptMeta.addModule(moduleScript, module.name, module.alias) - - // retrieve and return process from meta - return scriptMeta.getProcess(procKey) -} - -// use Reflection to get a ScriptParser / ScriptLoader -// <25.02.0-edge: new nextflow.script.ScriptParser(session) -// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) -def _getScriptLoader(nextflow.Session session) { - // try using the old method - try { - Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') - return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) - } catch (ClassNotFoundException e) { - // else try with the new method - try { - Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') - def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) - return createMethod.invoke(null, session) // null because create is static - } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { - // Handle the case where neither class is found - throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) - } - } -} - -// defaults -meta["defaults"] = [ - // key to be used to trace the process and determine output names - key: null, - - // fixed arguments to be passed to script - args: [:], - - // default directives - directives: readJsonBlob('''{ - "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/qc/calculate_qc_metrics", - "tag" : "disable-scrublet_build" - }, - "label" : [ - "singlecpu", - "midmem" - ], - "tag" : "$id" -}'''), - - // auto settings - auto: readJsonBlob('''{ - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false -}'''), - - // Apply a map over the incoming tuple - // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` - map: null, - - // Apply a map over the ID element of a tuple (i.e. the first element) - // Example: `{ id -> id + "_foo" }` - mapId: null, - - // Apply a map over the data element of a tuple (i.e. the second element) - // Example: `{ data -> [ input: data.output ] }` - mapData: null, - - // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) - // Example: `{ pt -> pt.drop(1) }` - mapPassthrough: null, - - // Filter the channel - // Example: `{ tup -> tup[0] == "foo" }` - filter: null, - - // Choose whether or not to run the component on the tuple if the condition is true. - // Otherwise, the tuple will be passed through. - // Example: `{ tup -> tup[0] != "skip_this" }` - runIf: null, - - // Rename keys in the data field of the tuple (i.e. the second element) - // Will likely be deprecated in favour of `fromState`. - // Example: `[ "new_key": "old_key" ]` - renameKeys: null, - - // Fetch data from the state and pass it to the module without altering the current state. - // - // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be passed to the module as is. - // - If it is a `List[String]`, the data will be the values of the state at the given keys. - // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. - // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. - // - // Example: `{ id, state -> [input: state.fastq_file] }` - // Default: `null` - fromState: null, - - // Determine how the state should be updated after the module has been run. - // - // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be replaced with the output of the module. - // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. - // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. - // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. - // - // Example: `{ id, output, state -> state + [counts: state.output] }` - // Default: `{ id, output, state -> output }` - toState: null, - - // Whether or not to print debug messages - // Default: `false` - debug: false -] - -// initialise default workflow -meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) - -// add workflow to environment -nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) - -// anonymous workflow for running this module as a standalone -workflow { - // add id argument if it's not already in the config - // TODO: deep copy - def newConfig = deepClone(meta.config) - def newParams = deepClone(params) - - def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} - if (!argsContainsId) { - def idArg = [ - 'name': '--id', - 'required': false, - 'type': 'string', - 'description': 'A unique id for every entry.', - 'multiple': false - ] - newConfig.arguments.add(0, idArg) - newConfig = processConfig(newConfig) - } - if (!newParams.containsKey("id")) { - newParams.id = "run" - } - - helpMessage(newConfig) - - channelFromParams(newParams, newConfig) - // make sure id is not in the state if id is not in the args - | map {id, state -> - if (!argsContainsId) { - [id, state.findAll{k, v -> k != "id"}] - } else { - [id, state] - } - } - | meta.workflow.run( - auto: [ publish: "state" ] - ) -} - -// END COMPONENT-SPECIFIC CODE diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow.config deleted file mode 100644 index eebede6..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow.config +++ /dev/null @@ -1,126 +0,0 @@ -manifest { - name = 'qc/calculate_qc_metrics' - mainScript = 'main.nf' - nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' - description = 'Add basic quality control metrics to an .h5mu file.\n\nThe metrics are comparable to what scanpy.pp.calculate_qc_metrics output,\nalthough they have slightly different names:\n\nVar metrics (name in this component -> name in scanpy):\n - pct_dropout -> pct_dropout_by_{expr_type}\n - num_nonzero_obs -> n_cells_by_{expr_type}\n - obs_mean -> mean_{expr_type}\n - total_counts -> total_{expr_type}\n\n Obs metrics:\n - num_nonzero_vars -> n_genes_by_{expr_type}\n - pct_{var_qc_metrics} -> pct_{expr_type}_{qc_var}\n - total_counts_{var_qc_metrics} -> total_{expr_type}_{qc_var}\n - pct_of_counts_in_top_{top_n_vars}_vars -> pct_{expr_type}_in_top_{n}_{var_type}\n - total_counts -> total_{expr_type}\n \n' - author = 'Dries Schaumont' -} - -process.container = 'nextflow/bash:latest' - -// detect tempdir -tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' -).toAbsolutePath() - -profiles { - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - docker { - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - singularity { - singularity.enabled = true - singularity.autoMounts = true - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - podman { - podman.enabled = true - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - shifter { - shifter.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - } - charliecloud { - charliecloud.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - } -} - -process{ - withLabel: mem1gb { memory = 1000000000.B } - withLabel: mem2gb { memory = 2000000000.B } - withLabel: mem5gb { memory = 5000000000.B } - withLabel: mem10gb { memory = 10000000000.B } - withLabel: mem20gb { memory = 20000000000.B } - withLabel: mem50gb { memory = 50000000000.B } - withLabel: mem100gb { memory = 100000000000.B } - withLabel: mem200gb { memory = 200000000000.B } - withLabel: mem500gb { memory = 500000000000.B } - withLabel: mem1tb { memory = 1000000000000.B } - withLabel: mem2tb { memory = 2000000000000.B } - withLabel: mem5tb { memory = 5000000000000.B } - withLabel: mem10tb { memory = 10000000000000.B } - withLabel: mem20tb { memory = 20000000000000.B } - withLabel: mem50tb { memory = 50000000000000.B } - withLabel: mem100tb { memory = 100000000000000.B } - withLabel: mem200tb { memory = 200000000000000.B } - withLabel: mem500tb { memory = 500000000000000.B } - withLabel: mem1gib { memory = 1073741824.B } - withLabel: mem2gib { memory = 2147483648.B } - withLabel: mem4gib { memory = 4294967296.B } - withLabel: mem8gib { memory = 8589934592.B } - withLabel: mem16gib { memory = 17179869184.B } - withLabel: mem32gib { memory = 34359738368.B } - withLabel: mem64gib { memory = 68719476736.B } - withLabel: mem128gib { memory = 137438953472.B } - withLabel: mem256gib { memory = 274877906944.B } - withLabel: mem512gib { memory = 549755813888.B } - withLabel: mem1tib { memory = 1099511627776.B } - withLabel: mem2tib { memory = 2199023255552.B } - withLabel: mem4tib { memory = 4398046511104.B } - withLabel: mem8tib { memory = 8796093022208.B } - withLabel: mem16tib { memory = 17592186044416.B } - withLabel: mem32tib { memory = 35184372088832.B } - withLabel: mem64tib { memory = 70368744177664.B } - withLabel: mem128tib { memory = 140737488355328.B } - withLabel: mem256tib { memory = 281474976710656.B } - withLabel: mem512tib { memory = 562949953421312.B } - withLabel: cpu1 { cpus = 1 } - withLabel: cpu2 { cpus = 2 } - withLabel: cpu5 { cpus = 5 } - withLabel: cpu10 { cpus = 10 } - withLabel: cpu20 { cpus = 20 } - withLabel: cpu50 { cpus = 50 } - withLabel: cpu100 { cpus = 100 } - withLabel: cpu200 { cpus = 200 } - withLabel: cpu500 { cpus = 500 } - withLabel: cpu1000 { cpus = 1000 } -} - -includeConfig("nextflow_labels.config") diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_params.yaml deleted file mode 100644 index 47537b9..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_params.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Inputs -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "raw_counts" - -# Metrics added to .obs -# var_qc_metrics: ["ercc,highly_variable,mitochondrial"] -# var_qc_metrics_fill_na_value: true -# top_n_vars: [123] -output_obs_num_nonzero_vars: "num_nonzero_vars" -output_obs_total_counts_vars: "total_counts" - -# Metrics added to .var -output_var_num_nonzero_obs: "num_nonzero_obs" -output_var_total_counts_obs: "total_counts" -output_var_obs_mean: "obs_mean" -output_var_pct_dropout: "pct_dropout" - -# Outputs -# output: "$id.$key.output.h5mu" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_schema.json b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_schema.json deleted file mode 100644 index f59dab1..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/nextflow_schema.json +++ /dev/null @@ -1,170 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "calculate_qc_metrics", - "description": "Add basic quality control metrics to an .h5mu file.\n\nThe metrics are comparable to what scanpy.pp.calculate_qc_metrics output,\nalthough they have slightly different names:\n\nVar metrics (name in this component -> name in scanpy):\n - pct_dropout -> pct_dropout_by_{expr_type}\n - num_nonzero_obs -> n_cells_by_{expr_type}\n - obs_mean -> mean_{expr_type}\n - total_counts -> total_{expr_type}\n\n Obs metrics:\n - num_nonzero_vars -> n_genes_by_{expr_type}\n - pct_{var_qc_metrics} -> pct_{expr_type}_{qc_var}\n - total_counts_{var_qc_metrics} -> total_{expr_type}_{qc_var}\n - pct_of_counts_in_top_{top_n_vars}_vars -> pct_{expr_type}_in_top_{n}_{var_type}\n - total_counts -> total_{expr_type}\n \n", - "type": "object", - "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, - "inputs": { - "title": "Inputs", - "type": "object", - "description": "No description", - "properties": { - "input": { - "type": "string", - "format": "path", - "exists": true, - "description": "Input h5mu file", - "help_text": "Type: `file`, multiple: `False`, required, direction: `input`, example: `\"input.h5mu\"`. " - }, - "modality": { - "type": "string", - "description": "Which modality from the input MuData file to process", - "help_text": "Type: `string`, multiple: `False`, default: `\"rna\"`. ", - "default": "rna" - }, - "layer": { - "type": "string", - "description": "Layer from modality to use as input data", - "help_text": "Type: `string`, multiple: `False`, example: `\"raw_counts\"`. " - } - } - }, - "outputs": { - "title": "Outputs", - "type": "object", - "description": "No description", - "properties": { - "output": { - "type": "string", - "format": "path", - "description": "Output h5mu file.", - "help_text": "Type: `file`, multiple: `False`, default: `\"$id.$key.output.h5mu\"`, direction: `output`, example: `\"output.h5mu\"`. ", - "default": "$id.$key.output.h5mu" - }, - "output_compression": { - "type": "string", - "description": "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", - "help_text": "Type: `string`, multiple: `False`, example: `\"gzip\"`, choices: ``gzip`, `lzf``. ", - "enum": [ - "gzip", - "lzf" - ] - } - } - }, - "metrics added to .obs": { - "title": "Metrics added to .obs", - "type": "object", - "description": "No description", - "properties": { - "var_qc_metrics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Keys to select a boolean (containing only True or False) column from .var.\nFor each cell, calculate the proportion of total values for genes which are labeled 'True', \ncompared to the total sum of the values for all genes.\n", - "help_text": "Type: `string`, multiple: `True`, example: `[\"ercc,highly_variable,mitochondrial\"]`. " - }, - "var_qc_metrics_fill_na_value": { - "type": "boolean", - "description": "Fill any 'NA' values found in the columns specified with --var_qc_metrics to 'True' or 'False'.\nas False.\n", - "help_text": "Type: `boolean`, multiple: `False`. " - }, - "top_n_vars": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "Number of top vars to be used to calculate cumulative proportions.\nIf not specified, proportions are not calculated", - "help_text": "Type: `integer`, multiple: `True`. " - }, - "output_obs_num_nonzero_vars": { - "type": "string", - "description": "Name of column in .obs describing, for each observation, the number of stored values\n(including explicit zeroes)", - "help_text": "Type: `string`, multiple: `False`, default: `\"num_nonzero_vars\"`. ", - "default": "num_nonzero_vars" - }, - "output_obs_total_counts_vars": { - "type": "string", - "description": "Name of the column for .obs describing, for each observation (row),\nthe sum of the stored values in the columns.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"total_counts\"`. ", - "default": "total_counts" - } - } - }, - "metrics added to .var": { - "title": "Metrics added to .var", - "type": "object", - "description": "No description", - "properties": { - "output_var_num_nonzero_obs": { - "type": "string", - "description": "Name of column describing, for each feature, the number of stored values\n(including explicit zeroes)", - "help_text": "Type: `string`, multiple: `False`, default: `\"num_nonzero_obs\"`. ", - "default": "num_nonzero_obs" - }, - "output_var_total_counts_obs": { - "type": "string", - "description": "Name of the column in .var describing, for each feature (column),\nthe sum of the stored values in the rows.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"total_counts\"`. ", - "default": "total_counts" - }, - "output_var_obs_mean": { - "type": "string", - "description": "Name of the column in .obs providing the mean of the values in each row.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"obs_mean\"`. ", - "default": "obs_mean" - }, - "output_var_pct_dropout": { - "type": "string", - "description": "Name of the column in .obs providing for each feature the percentage of\nobservations the feature does not appear on (i.e", - "help_text": "Type: `string`, multiple: `False`, default: `\"pct_dropout\"`. ", - "default": "pct_dropout" - } - } - }, - "nextflow input-output arguments": { - "title": "Nextflow input-output arguments", - "type": "object", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "properties": { - "publish_dir": { - "type": "string", - "description": "Path to an output directory.", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " - } - } - } - }, - "allOf": [ - { - "$ref": "#/$defs/inputs" - }, - { - "$ref": "#/$defs/outputs" - }, - { - "$ref": "#/$defs/metrics added to .obs" - }, - { - "$ref": "#/$defs/metrics added to .var" - }, - { - "$ref": "#/$defs/nextflow input-output arguments" - } - ] -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_params.yaml deleted file mode 100644 index 635d850..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_params.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "prot" -# output: "output.h5mu" -# input_layer: "foo" -# output_layer: "foo" -axis: 0 -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_params.yaml deleted file mode 100644 index c945e51..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_params.yaml +++ /dev/null @@ -1,11 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -layer: # please fill in - example: ["foo"] -# output: "output.h5mu" -missing_ok: false -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_params.yaml deleted file mode 100644 index 50de330..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_params.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# input_layer: "foo" -# output_layer: "foo" -# output: "output.h5mu" -# base: 2.0 -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/compress_h5mu.py b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/compress_h5mu.py deleted file mode 100644 index 4b363ee..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/compress_h5mu.py +++ /dev/null @@ -1,87 +0,0 @@ -import shutil -from anndata import AnnData -from mudata import write_h5ad -from h5py import File as H5File -from h5py import Group, Dataset -from pathlib import Path -from typing import Union, Literal -from functools import partial - - -def compress_h5mu( - input_path: Union[str, Path], - output_path: Union[str, Path], - compression: Union[Literal["gzip"], Literal["lzf"]], -): - input_path, output_path = str(input_path), str(output_path) - - def copy_attributes(in_object, out_object): - for key, value in in_object.attrs.items(): - out_object.attrs[key] = value - - def visit_path( - output_h5: H5File, - compression: Union[Literal["gzip"], Literal["lzf"]], - name: str, - object: Union[Group, Dataset], - ): - if isinstance(object, Group): - new_group = output_h5.create_group(name) - copy_attributes(object, new_group) - elif isinstance(object, Dataset): - # Compression only works for non-scalar Dataset objects - # Scalar objects dont have a shape defined - if not object.compression and object.shape not in [None, ()]: - new_dataset = output_h5.create_dataset( - name, data=object, compression=compression - ) - copy_attributes(object, new_dataset) - else: - output_h5.copy(object, name) - else: - raise NotImplementedError( - f"Could not copy element {name}, " - f"type has not been implemented yet: {type(object)}" - ) - - with ( - H5File(input_path, "r") as input_h5, - H5File(output_path, "w", userblock_size=512) as output_h5, - ): - copy_attributes(input_h5, output_h5) - input_h5.visititems(partial(visit_path, output_h5, compression)) - - with open(input_path, "rb") as input_bytes: - # Mudata puts metadata like this in the first 512 bytes: - # MuData (format-version=0.1.0;creator=muon;creator-version=0.2.0) - # See mudata/_core/io.py, read_h5mu() function - starting_metadata = input_bytes.read(100) - # The metadata is padded with extra null bytes up until 512 bytes - truncate_location = starting_metadata.find(b"\x00") - starting_metadata = starting_metadata[:truncate_location] - with open(output_path, "br+") as f: - nbytes = f.write(starting_metadata) - f.write(b"\0" * (512 - nbytes)) - - -def write_h5ad_to_h5mu_with_compression( - output_file: Union[str, Path], - h5mu: Union[str, Path], - modality_name: str, - modality_data: AnnData, - output_compression=None, -): - output_file = Path(output_file) - h5mu = Path(h5mu) - output_file_uncompressed = ( - output_file.with_name(output_file.stem + "_uncompressed.h5mu") - if output_compression - else output_file - ) - shutil.copyfile(h5mu, output_file_uncompressed) - write_h5ad(filename=output_file_uncompressed, mod=modality_name, data=modality_data) - if output_compression: - compress_h5mu( - output_file_uncompressed, output_file, compression=output_compression - ) - output_file_uncompressed.unlink() diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_params.yaml deleted file mode 100644 index 366375f..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_params.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# input_layer: "foo" -# output: "output.h5mu" -# output_layer: "foo" -# target_sum: 123 -exclude_highly_expressed: false -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/setup_logger.py b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/setup_logger.py deleted file mode 100644 index 3ca1cdb..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/setup_logger.py +++ /dev/null @@ -1,12 +0,0 @@ -def setup_logger(): - import logging - from sys import stdout - - logger = logging.getLogger() - logger.setLevel(logging.INFO) - console_handler = logging.StreamHandler(stdout) - logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s") - console_handler.setFormatter(logFormatter) - logger.addHandler(console_handler) - - return logger diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/compress_h5mu.py b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/compress_h5mu.py deleted file mode 100644 index 4b363ee..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/compress_h5mu.py +++ /dev/null @@ -1,87 +0,0 @@ -import shutil -from anndata import AnnData -from mudata import write_h5ad -from h5py import File as H5File -from h5py import Group, Dataset -from pathlib import Path -from typing import Union, Literal -from functools import partial - - -def compress_h5mu( - input_path: Union[str, Path], - output_path: Union[str, Path], - compression: Union[Literal["gzip"], Literal["lzf"]], -): - input_path, output_path = str(input_path), str(output_path) - - def copy_attributes(in_object, out_object): - for key, value in in_object.attrs.items(): - out_object.attrs[key] = value - - def visit_path( - output_h5: H5File, - compression: Union[Literal["gzip"], Literal["lzf"]], - name: str, - object: Union[Group, Dataset], - ): - if isinstance(object, Group): - new_group = output_h5.create_group(name) - copy_attributes(object, new_group) - elif isinstance(object, Dataset): - # Compression only works for non-scalar Dataset objects - # Scalar objects dont have a shape defined - if not object.compression and object.shape not in [None, ()]: - new_dataset = output_h5.create_dataset( - name, data=object, compression=compression - ) - copy_attributes(object, new_dataset) - else: - output_h5.copy(object, name) - else: - raise NotImplementedError( - f"Could not copy element {name}, " - f"type has not been implemented yet: {type(object)}" - ) - - with ( - H5File(input_path, "r") as input_h5, - H5File(output_path, "w", userblock_size=512) as output_h5, - ): - copy_attributes(input_h5, output_h5) - input_h5.visititems(partial(visit_path, output_h5, compression)) - - with open(input_path, "rb") as input_bytes: - # Mudata puts metadata like this in the first 512 bytes: - # MuData (format-version=0.1.0;creator=muon;creator-version=0.2.0) - # See mudata/_core/io.py, read_h5mu() function - starting_metadata = input_bytes.read(100) - # The metadata is padded with extra null bytes up until 512 bytes - truncate_location = starting_metadata.find(b"\x00") - starting_metadata = starting_metadata[:truncate_location] - with open(output_path, "br+") as f: - nbytes = f.write(starting_metadata) - f.write(b"\0" * (512 - nbytes)) - - -def write_h5ad_to_h5mu_with_compression( - output_file: Union[str, Path], - h5mu: Union[str, Path], - modality_name: str, - modality_data: AnnData, - output_compression=None, -): - output_file = Path(output_file) - h5mu = Path(h5mu) - output_file_uncompressed = ( - output_file.with_name(output_file.stem + "_uncompressed.h5mu") - if output_compression - else output_file - ) - shutil.copyfile(h5mu, output_file_uncompressed) - write_h5ad(filename=output_file_uncompressed, mod=modality_name, data=modality_data) - if output_compression: - compress_h5mu( - output_file_uncompressed, output_file, compression=output_compression - ) - output_file_uncompressed.unlink() diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_params.yaml deleted file mode 100644 index 15590bf..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_params.yaml +++ /dev/null @@ -1,13 +0,0 @@ -# Arguments -input: # please fill in - example: "input.h5mu" -modality: "rna" -# input_layer: "foo" -# output_layer: "foo" -# max_value: 123.0 -zero_center: true -# output: "output.h5mu" -# output_compression: "gzip" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/setup_logger.py b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/setup_logger.py deleted file mode 100644 index 3ca1cdb..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/setup_logger.py +++ /dev/null @@ -1,12 +0,0 @@ -def setup_logger(): - import logging - from sys import stdout - - logger = logging.getLogger() - logger.setLevel(logging.INFO) - console_handler = logging.StreamHandler(stdout) - logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s") - console_handler.setFormatter(logFormatter) - logger.addHandler(console_handler) - - return logger diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_params.yaml deleted file mode 100644 index 3ffadcb..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_params.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Input -id: # please fill in - example: "foo" -input: # please fill in - example: "dataset.h5mu" -# layer: "foo" - -# Output -# output: "$id.$key.output.h5mu" - -# Filtering options -# min_counts: 200 -# max_counts: 5000000 -# min_guides_per_cell: 200 -# max_guides_per_cell: 1500000 -# min_cells_per_guide: 3 - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_params.yaml deleted file mode 100644 index ca85468..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_params.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Inputs -id: # please fill in - example: "foo" -input: # please fill in - example: "dataset.h5mu" -layer: "log_normalized" -modality: "rna" - -# Outputs -# output: "$id.$key.output.h5mu" - -# PCA options -obsm_pca: "X_pca" -# var_pca_feature_selection: "foo" -# pca_loadings_varm_output: "foo" -# pca_variance_uns_output: "foo" -pca_overwrite: false - -# Neighbour calculation -uns_neighbors: "neighbors" -obsp_neighbor_distances: "distances" -obsp_neighbor_connectivities: "connectivities" - -# Umap options -obsm_umap: "X_umap" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_params.yaml deleted file mode 100644 index 9cb208b..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_params.yaml +++ /dev/null @@ -1,25 +0,0 @@ -# Inputs -input: # please fill in - example: "dataset.h5mu" -obsm_input: # please fill in - example: "foo" -modality: "rna" - -# Outputs -# output: "$id.$key.output.h5mu" - -# Neighbour calculation -uns_neighbors: # please fill in - example: "foo" -obsp_neighbor_distances: # please fill in - example: "foo" -obsp_neighbor_connectivities: # please fill in - example: "foo" - -# Clustering options -# obs_cluster: "foo" -leiden_resolution: [1.0] - -# Umap options -# obsm_umap: "foo" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_params.yaml deleted file mode 100644 index 38f6e34..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_params.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Inputs -id: # please fill in - example: "foo" -input: # please fill in - example: ["input.h5mu"] -# rna_layer: "foo" -# prot_layer: "foo" - -# Outputs -# output: "$id.$key.output.h5mu" - -# Highly variable features detection -highly_variable_features_var_output: "filter_with_hvg" -highly_variable_features_obs_batch_key: "sample_id" - -# QC metrics calculation options -var_qc_metrics: ["filter_with_hvg"] -top_n_vars: [50, 100, 200, 500] - -# PCA options -pca_overwrite: false - -# CLR options -clr_axis: 0 - -# RNA Scaling options -rna_enable_scaling: false -rna_scaling_output_layer: "scaled" -rna_scaling_pca_obsm_output: "scaled_pca" -rna_scaling_pca_loadings_varm_output: "scaled_pca_loadings" -rna_scaling_pca_variance_uns_output: "scaled_pca_variance" -rna_scaling_umap_obsm_output: "scaled_umap" -# rna_scaling_max_value: 123.0 -rna_scaling_zero_center: true - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_params.yaml deleted file mode 100644 index a930bfa..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_params.yaml +++ /dev/null @@ -1,79 +0,0 @@ -# Inputs -id: # please fill in - example: "foo" -input: # please fill in - example: "input.h5mu" -# rna_layer: "foo" -# prot_layer: "foo" -# gdo_layer: "foo" - -# Outputs -# output: "$id.$key.output.h5mu" - -# Sample ID options -add_id_to_obs: true -add_id_obs_output: "sample_id" -add_id_make_observation_keys_unique: true - -# RNA filtering options -# rna_min_counts: 200 -# rna_max_counts: 5000000 -# rna_min_genes_per_cell: 200 -# rna_max_genes_per_cell: 1500000 -# rna_min_cells_per_gene: 3 -# rna_min_fraction_mito: 0.0 -# rna_max_fraction_mito: 0.2 -# rna_min_fraction_ribo: 0.0 -# rna_max_fraction_ribo: 0.2 -skip_scrublet_filtering: false - -# CITE-seq filtering options -# prot_min_counts: 3 -# prot_max_counts: 5000000 -# prot_min_proteins_per_cell: 200 -# prot_max_proteins_per_cell: 100000000 -# prot_min_cells_per_protein: 3 - -# GDO filtering options -# gdo_min_counts: 3 -# gdo_max_counts: 5000000 -# gdo_min_guides_per_cell: 200 -# gdo_max_guides_per_cell: 100000000 -# gdo_min_cells_per_guide: 3 - -# Highly variable features detection -highly_variable_features_var_output: "filter_with_hvg" -highly_variable_features_obs_batch_key: "sample_id" - -# Mitochondrial & Ribosomal Gene Detection -# var_gene_names: "gene_symbol" -# var_name_mitochondrial_genes: "foo" -# obs_name_mitochondrial_fraction: "foo" -mitochondrial_gene_regex: "^[mM][tT]-" -# var_name_ribosomal_genes: "foo" -# obs_name_ribosomal_fraction: "foo" -ribosomal_gene_regex: "^[Mm]?[Rr][Pp][LlSs]" - -# QC metrics calculation options -# var_qc_metrics: ["ercc,highly_variable"] -top_n_vars: [50, 100, 200, 500] - -# PCA options -pca_overwrite: false - -# CLR options -clr_axis: 0 - -# RNA Scaling options -rna_enable_scaling: false -rna_scaling_output_layer: "scaled" -rna_scaling_pca_obsm_output: "scaled_pca" -rna_scaling_pca_loadings_varm_output: "scaled_pca_loadings" -rna_scaling_pca_variance_uns_output: "scaled_pca_variance" -rna_scaling_umap_obsm_output: "scaled_umap" -# rna_scaling_max_value: 123.0 -rna_scaling_zero_center: true - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_params.yaml deleted file mode 100644 index 2511f2d..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_params.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Inputs -id: # please fill in - example: "concatenated" -input: # please fill in - example: "dataset.h5mu" -# layer: "foo" - -# Outputs -# output: "$id.$key.output.h5mu" - -# QC metrics calculation options -# var_qc_metrics: ["ercc,highly_variable"] -top_n_vars: [50, 100, 200, 500] -output_obs_num_nonzero_vars: "num_nonzero_vars" -output_obs_total_counts_vars: "total_counts" -output_var_num_nonzero_obs: "num_nonzero_obs" -output_var_total_counts_obs: "total_counts" -output_var_obs_mean: "obs_mean" -output_var_pct_dropout: "pct_dropout" - -# CLR arguments -clr_axis: 0 - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_params.yaml deleted file mode 100644 index 775308b..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_params.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Input -id: # please fill in - example: "foo" -input: # please fill in - example: "dataset.h5mu" -# layer: "foo" - -# Output -# output: "$id.$key.output.h5mu" - -# Filtering options -# min_counts: 200 -# max_counts: 5000000 -# min_proteins_per_cell: 200 -# max_proteins_per_cell: 1500000 -# min_cells_per_protein: 3 - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/.config.vsh.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/.config.vsh.yaml deleted file mode 100644 index b9d3367..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/.config.vsh.yaml +++ /dev/null @@ -1,406 +0,0 @@ -name: "qc" -namespace: "workflows/qc" -version: "disable-scrublet_build" -authors: -- name: "Dries Schaumont" - roles: - - "author" - - "maintainer" - info: - role: "Core Team Member" - links: - email: "dries@data-intuitive.com" - github: "DriesSchaumont" - orcid: "0000-0002-4389-0440" - linkedin: "dries-schaumont" - organizations: - - name: "Data Intuitive" - href: "https://www.data-intuitive.com" - role: "Data Scientist" -argument_groups: -- name: "Inputs" - arguments: - - type: "string" - name: "--id" - description: "ID of the sample." - info: null - example: - - "foo" - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "file" - name: "--input" - alternatives: - - "-i" - description: "Path to the sample." - info: null - example: - - "input.h5mu" - must_exist: true - create_parent: true - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--modality" - description: "Which modality to process." - info: null - default: - - "rna" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--layer" - description: "Layer to calculate qc metrics for." - info: null - example: - - "raw_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Mitochondrial & Ribosomal Gene Detection" - arguments: - - type: "string" - name: "--var_gene_names" - description: ".var column name to be used to detect mitochondrial/ribosomal genes\ - \ instead of .var_names (default if not set).\nGene names matching with the\ - \ regex value from --mitochondrial_gene_regex or --ribosomal_gene_regex will\ - \ be \nidentified as mitochondrial or ribosomal genes, respectively.\n" - info: null - example: - - "gene_symbol" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--var_name_mitochondrial_genes" - description: "In which .var slot to store a boolean array corresponding the mitochondrial\ - \ genes.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--obs_name_mitochondrial_fraction" - description: ".Obs slot to store the fraction of reads found to be mitochondrial.\ - \ Defaults to 'fraction_' suffixed by the value of --var_name_mitochondrial_genes\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--mitochondrial_gene_regex" - description: "Regex string that identifies mitochondrial genes from --var_gene_names.\n\ - By default will detect human and mouse mitochondrial genes from a gene symbol.\n" - info: null - default: - - "^[mM][tT]-" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--var_name_ribosomal_genes" - description: "In which .var slot to store a boolean array corresponding the ribosomal\ - \ genes.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--obs_name_ribosomal_fraction" - description: "When specified, write the fraction of counts originating from ribosomal\ - \ genes \n(based on --ribosomal_gene_regex) to an .obs column with the specified\ - \ name.\nRequires --var_name_ribosomal_genes.\n" - info: null - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--ribosomal_gene_regex" - description: "Regex string that identifies ribosomal genes from --var_gene_names.\n\ - By default will detect human and mouse ribosomal genes from a gene symbol.\n" - info: null - default: - - "^[Mm]?[Rr][Pp][LlSs]" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "QC metrics calculation options" - arguments: - - type: "string" - name: "--var_qc_metrics" - description: "Keys to select a boolean (containing only True or False) column\ - \ from .var.\nFor each cell, calculate the proportion of total values for genes\ - \ which are labeled 'True', \ncompared to the total sum of the values for all\ - \ genes. Defaults to the value from\n--var_name_mitochondrial_genes.\n" - info: null - example: - - "ercc,highly_variable" - required: false - direction: "input" - multiple: true - multiple_sep: "," - - type: "integer" - name: "--top_n_vars" - description: "Number of top vars to be used to calculate cumulative proportions.\n\ - If not specified, proportions are not calculated. `--top_n_vars 20,50` finds\n\ - cumulative proportion to the 20th and 50th most expressed vars.\n" - info: null - default: - - 50 - - 100 - - 200 - - 500 - required: false - direction: "input" - multiple: true - multiple_sep: "," - - type: "string" - name: "--output_obs_num_nonzero_vars" - description: "Name of column in .obs describing, for each observation, the number\ - \ of stored values\n(including explicit zeroes). In other words, the name of\ - \ the column that counts\nfor each row the number of columns that contain data.\n" - info: null - default: - - "num_nonzero_vars" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_obs_total_counts_vars" - description: "Name of the column for .obs describing, for each observation (row),\n\ - the sum of the stored values in the columns.\n" - info: null - default: - - "total_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_num_nonzero_obs" - description: "Name of column describing, for each feature, the number of stored\ - \ values\n(including explicit zeroes). In other words, the name of the column\ - \ that counts\nfor each column the number of rows that contain data.\n" - info: null - default: - - "num_nonzero_obs" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_total_counts_obs" - description: "Name of the column in .var describing, for each feature (column),\n\ - the sum of the stored values in the rows.\n" - info: null - default: - - "total_counts" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_obs_mean" - description: "Name of the column in .obs providing the mean of the values in each\ - \ row.\n" - info: null - default: - - "obs_mean" - required: false - direction: "input" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_var_pct_dropout" - description: "Name of the column in .obs providing for each feature the percentage\ - \ of\nobservations the feature does not appear on (i.e. is missing). Same as\ - \ `--output_var_num_nonzero_obs`\nbut percentage based.\n" - info: null - default: - - "pct_dropout" - required: false - direction: "input" - multiple: false - multiple_sep: ";" -- name: "Outputs" - arguments: - - type: "file" - name: "--output" - description: "Destination path to the output." - info: null - example: - - "output.h5mu" - must_exist: true - create_parent: true - required: true - direction: "output" - multiple: false - multiple_sep: ";" -resources: -- type: "nextflow_script" - path: "main.nf" - is_executable: true - entrypoint: "run_wf" -- type: "file" - path: "utils" -- type: "file" - path: "nextflow_labels.config" - dest: "nextflow_labels.config" -description: "A pipeline to add basic qc statistics to a MuData " -test_resources: -- type: "nextflow_script" - path: "test.nf" - is_executable: true - entrypoint: "test_wf" -- type: "file" - path: "concat_test_data" -- type: "file" - path: "pbmc_1k_protein_v3" -info: - test_dependencies: - - name: "qc_test" - namespace: "test_workflows/qc" -status: "enabled" -scope: - image: "public" - target: "public" -dependencies: -- name: "metadata/grep_annotation_column" - repository: - type: "local" -- name: "qc/calculate_qc_metrics" - repository: - type: "local" -license: "MIT" -links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" -runners: -- type: "nextflow" - id: "nextflow" - directives: - tag: "$id" - auto: - simplifyInput: true - simplifyOutput: false - transcript: false - publish: false - config: - labels: - mem1gb: "memory = 1000000000.B" - mem2gb: "memory = 2000000000.B" - mem5gb: "memory = 5000000000.B" - mem10gb: "memory = 10000000000.B" - mem20gb: "memory = 20000000000.B" - mem50gb: "memory = 50000000000.B" - mem100gb: "memory = 100000000000.B" - mem200gb: "memory = 200000000000.B" - mem500gb: "memory = 500000000000.B" - mem1tb: "memory = 1000000000000.B" - mem2tb: "memory = 2000000000000.B" - mem5tb: "memory = 5000000000000.B" - mem10tb: "memory = 10000000000000.B" - mem20tb: "memory = 20000000000000.B" - mem50tb: "memory = 50000000000000.B" - mem100tb: "memory = 100000000000000.B" - mem200tb: "memory = 200000000000000.B" - mem500tb: "memory = 500000000000000.B" - mem1gib: "memory = 1073741824.B" - mem2gib: "memory = 2147483648.B" - mem4gib: "memory = 4294967296.B" - mem8gib: "memory = 8589934592.B" - mem16gib: "memory = 17179869184.B" - mem32gib: "memory = 34359738368.B" - mem64gib: "memory = 68719476736.B" - mem128gib: "memory = 137438953472.B" - mem256gib: "memory = 274877906944.B" - mem512gib: "memory = 549755813888.B" - mem1tib: "memory = 1099511627776.B" - mem2tib: "memory = 2199023255552.B" - mem4tib: "memory = 4398046511104.B" - mem8tib: "memory = 8796093022208.B" - mem16tib: "memory = 17592186044416.B" - mem32tib: "memory = 35184372088832.B" - mem64tib: "memory = 70368744177664.B" - mem128tib: "memory = 140737488355328.B" - mem256tib: "memory = 281474976710656.B" - mem512tib: "memory = 562949953421312.B" - cpu1: "cpus = 1" - cpu2: "cpus = 2" - cpu5: "cpus = 5" - cpu10: "cpus = 10" - cpu20: "cpus = 20" - cpu50: "cpus = 50" - cpu100: "cpus = 100" - cpu200: "cpus = 200" - cpu500: "cpus = 500" - cpu1000: "cpus = 1000" - script: - - "includeConfig(\"nextflow_labels.config\")" - debug: false - container: "docker" -build_info: - config: "src/workflows/qc/qc/config.vsh.yaml" - runner: "nextflow" - engine: "native" - output: "target/nextflow/workflows/qc/qc" - executable: "target/nextflow/workflows/qc/qc/main.nf" - viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" - git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" - dependencies: - - "target/nextflow/metadata/grep_annotation_column" - - "target/nextflow/qc/calculate_qc_metrics" -package_config: - name: "openpipeline" - summary: "Best-practice workflows for single-cell multi-omics analyses.\n" - description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ - \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ - \nIn terms of workflows, the following has been made available, but keep in mind\ - \ that\nindividual tools and functionality can be executed as standalone components\ - \ as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n\ - \ * Ingestion: Read mapping and generating a count matrix.\n * Single sample\ - \ processing: cell filtering and doublet detection.\n * Multisample processing:\ - \ Count transformation, normalization, QC metric calulations.\n * Integration:\ - \ Clustering, integration and batch correction using single and multimodal methods.\n\ - \ * Downstream analysis workflows\n" - info: - test_resources: - - type: "s3" - path: "s3://openpipelines-data" - dest: "resources_test" - viash_version: "0.9.4" - source: "src" - target: "target" - config_mods: - - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ - .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ - )'" - - ".version := \"disable-scrublet_build\"" - keywords: - - "single-cell" - - "multimodal" - license: "MIT" - organization: "openpipelines-bio" - links: - repository: "https://github.com/openpipelines-bio/openpipeline" - docker_registry: "ghcr.io" - homepage: "https://openpipelines.bio" - documentation: "https://openpipelines.bio/fundamentals" - issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/main.nf b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/main.nf deleted file mode 100644 index d0f070f..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/main.nf +++ /dev/null @@ -1,3758 +0,0 @@ -// qc disable-scrublet_build -// -// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative -// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data -// Intuitive. -// -// The component may contain files which fall under a different license. The -// authors of this component should specify the license in the header of such -// files, or include a separate license file detailing the licenses of all included -// files. -// -// Component authors: -// * Dries Schaumont (author, maintainer) - -//////////////////////////// -// VDSL3 helper functions // -//////////////////////////// - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' -class UnexpectedArgumentTypeException extends Exception { - String errorIdentifier - String stage - String plainName - String expectedClass - String foundClass - - // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} - UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { - super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + - "Expected type: ${expectedClass}. Found type: ${foundClass}") - this.errorIdentifier = errorIdentifier - this.stage = stage - this.plainName = plainName - this.expectedClass = expectedClass - this.foundClass = foundClass - } -} - -/** - * Checks if the given value is of the expected type. If not, an exception is thrown. - * - * @param stage The stage of the argument (input or output) - * @param par The parameter definition - * @param value The value to check - * @param errorIdentifier The identifier to use in the error message - * @return The value, if it is of the expected type - * @throws UnexpectedArgumentTypeException If the value is not of the expected type -*/ -def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { - // expectedClass will only be != null if value is not of the expected type - def expectedClass = null - def foundClass = null - - // todo: split if need be - - if (!par.required && value == null) { - expectedClass = null - } else if (par.multiple) { - if (value !instanceof Collection) { - value = [value] - } - - // split strings - value = value.collectMany{ val -> - if (val instanceof String) { - // collect() to ensure that the result is a List and not simply an array - val.split(par.multiple_sep).collect() - } else { - [val] - } - } - - // process globs - if (par.type == "file" && par.direction == "input") { - value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() - } - - // check types of elements in list - try { - value = value.collect { listVal -> - _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) - } - } catch (UnexpectedArgumentTypeException e) { - expectedClass = "List[${e.expectedClass}]" - foundClass = "List[${e.foundClass}]" - } - } else if (par.type == "string") { - // cast to string if need be. only cast if the value is a GString - if (value instanceof GString) { - value = value as String - } - expectedClass = value instanceof String ? null : "String" - } else if (par.type == "integer") { - // cast to integer if need be - if (value !instanceof Integer) { - try { - value = value as Integer - } catch (NumberFormatException e) { - expectedClass = "Integer" - } - } - } else if (par.type == "long") { - // cast to long if need be - if (value !instanceof Long) { - try { - value = value as Long - } catch (NumberFormatException e) { - expectedClass = "Long" - } - } - } else if (par.type == "double") { - // cast to double if need be - if (value !instanceof Double) { - try { - value = value as Double - } catch (NumberFormatException e) { - expectedClass = "Double" - } - } - } else if (par.type == "float") { - // cast to float if need be - if (value !instanceof Float) { - try { - value = value as Float - } catch (NumberFormatException e) { - expectedClass = "Float" - } - } - } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { - // cast to boolean if need be - if (value !instanceof Boolean) { - try { - value = value as Boolean - } catch (Exception e) { - expectedClass = "Boolean" - } - } - } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { - // cast to path if need be - if (value instanceof String) { - value = file(value, hidden: true) - } - if (value instanceof File) { - value = value.toPath() - } - expectedClass = value instanceof Path ? null : "Path" - } else if (par.type == "file" && stage == "input" && par.direction == "output") { - // cast to string if need be - if (value !instanceof String) { - try { - value = value as String - } catch (Exception e) { - expectedClass = "String" - } - } - } else { - // didn't find a match for par.type - expectedClass = par.type - } - - if (expectedClass != null) { - if (foundClass == null) { - foundClass = value.getClass().getName() - } - throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) - } - - return value -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' -Map _processInputValues(Map inputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.required && arg.direction == "input") { - assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" - } - } - - inputs = inputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" - - value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return inputs -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' -Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - outputs = outputs.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && it.direction == "output" } - assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" - - value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") - - [ name, value ] - } - } - return outputs -} - -void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { - if (!workflow.stubRun) { - config.allArguments.each { arg -> - if (arg.direction == "output" && arg.required) { - assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : - "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" - } - } - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' -class IDChecker { - final def items = [] as Set - - @groovy.transform.WithWriteLock - boolean observe(String item) { - if (items.contains(item)) { - return false - } else { - items << item - return true - } - } - - @groovy.transform.WithReadLock - boolean contains(String item) { - return items.contains(item) - } - - @groovy.transform.WithReadLock - Set getItems() { - return items.clone() - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' - -/** - * Check if the ids are unique across parameter sets - * - * @param parameterSets a list of parameter sets. - */ -private void _checkUniqueIds(List>> parameterSets) { - def ppIds = parameterSets.collect{it[0]} - assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' - -// helper functions for reading params from file // -def _getChild(parent, child) { - if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { - child - } else { - def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() - parentAbsolute.replaceAll('/[^/]*$', "/") + child - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' -/** - * Figure out the param list format based on the file extension - * - * @param param_list A String containing the path to the parameter list file. - * - * @return A String containing the format of the parameter list file. - */ -def _paramListGuessFormat(param_list) { - if (param_list !instanceof String) { - "asis" - } else if (param_list.endsWith(".csv")) { - "csv" - } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { - "json" - } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { - "yaml" - } else { - "yaml_blob" - } -} - - -/** - * Read the param list - * - * @param param_list One of the following: - * - A String containing the path to the parameter list file (csv, json or yaml), - * - A yaml blob of a list of maps (yaml_blob), - * - Or a groovy list of maps (asis). - * @param config A Map of the Viash configuration. - * - * @return A List of Maps containing the parameters. - */ -def _parseParamList(param_list, Map config) { - // first determine format by extension - def paramListFormat = _paramListGuessFormat(param_list) - - def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? - file(param_list, hidden: true) : - null - - // get the correct parser function for the detected params_list format - def paramSets = [] - if (paramListFormat == "asis") { - paramSets = param_list - } else if (paramListFormat == "yaml_blob") { - paramSets = readYamlBlob(param_list) - } else if (paramListFormat == "yaml") { - paramSets = readYaml(paramListPath) - } else if (paramListFormat == "json") { - paramSets = readJson(paramListPath) - } else if (paramListFormat == "csv") { - paramSets = readCsv(paramListPath) - } else { - error "Format of provided --param_list not recognised.\n" + - "Found: '$paramListFormat'.\n" + - "Expected: a csv file, a json file, a yaml file,\n" + - "a yaml blob or a groovy list of maps." - } - - // data checks - assert paramSets instanceof List: "--param_list should contain a list of maps" - for (value in paramSets) { - assert value instanceof Map: "--param_list should contain a list of maps" - } - - // id is argument - def idIsArgument = config.allArguments.any{it.plainName == "id"} - - // Reformat from List to List> by adding the ID as first element of a Tuple2 - paramSets = paramSets.collect({ data -> - def id = data.id - if (!idIsArgument) { - data = data.findAll{k, v -> k != "id"} - } - [id, data] - }) - - // Split parameters with 'multiple: true' - paramSets = paramSets.collect({ id, data -> - data = _splitParams(data, config) - [id, data] - }) - - // The paths of input files inside a param_list file may have been specified relatively to the - // location of the param_list file. These paths must be made absolute. - if (paramListPath) { - paramSets = paramSets.collect({ id, data -> - def new_data = data.collectEntries{ parName, parValue -> - def par = config.allArguments.find{it.plainName == parName} - if (par && par.type == "file" && par.direction == "input") { - if (parValue instanceof Collection) { - parValue = parValue.collectMany{path -> - def x = _resolveSiblingIfNotAbsolute(path, paramListPath) - x instanceof Collection ? x : [x] - } - } else { - parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) - } - } - [parName, parValue] - } - [id, new_data] - }) - } - - return paramSets -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' -/** - * Split parameters for arguments that accept multiple values using their separator - * - * @param paramList A Map containing parameters to split. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A Map of parameters where the parameter values have been split into a list using - * their seperator. - */ -Map _splitParams(Map parValues, Map config){ - def parsedParamValues = parValues.collectEntries { parName, parValue -> - def parameterSettings = config.allArguments.find({it.plainName == parName}) - - if (!parameterSettings) { - // if argument is not found, do not alter - return [parName, parValue] - } - if (parameterSettings.multiple) { // Check if parameter can accept multiple values - if (parValue instanceof Collection) { - parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } - } else if (parValue instanceof String) { - parValue = parValue.split(parameterSettings.multiple_sep) - } else if (parValue == null) { - parValue = [] - } else { - parValue = [ parValue ] - } - parValue = parValue.flatten() - } - // For all parameters check if multiple values are only passed for - // arguments that allow it. Quietly simplify lists of length 1. - if (!parameterSettings.multiple && parValue instanceof Collection) { - assert parValue.size() == 1 : - "Error: argument ${parName} has too many values.\n" + - " Expected amount: 1. Found: ${parValue.size()}" - parValue = parValue[0] - } - [parName, parValue] - } - return parsedParamValues -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' -/** - * Parse nextflow parameters based on settings defined in a viash config. - * Return a list of parameter sets, each parameter set corresponding to - * an event in a nextflow channel. The output from this function can be used - * with Channel.fromList to create a nextflow channel with Vdsl3 formatted - * events. - * - * This function performs: - * - A filtering of the params which can be found in the config file. - * - Process the params_list argument which allows a user to to initialise - * a Vsdl3 channel with multiple parameter sets. Possible formats are - * csv, json, yaml, or simply a yaml_blob. A csv should have column names - * which correspond to the different arguments of this pipeline. A json or a yaml - * file should be a list of maps, each of which has keys corresponding to the - * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. - * When passing a csv, json or yaml, relative path names are relativized to the - * location of the parameter file. - * - Combine the parameter sets into a vdsl3 Channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A list of parameters with the first element of the event being - * the event ID and the second element containing a map of the parsed parameters. - */ - -private List>> _paramsToParamSets(Map params, Map config){ - // todo: fetch key from run args - def key_ = config.name - - /* parse regular parameters (not in param_list) */ - /*************************************************/ - def globalParams = config.allArguments - .findAll { params.containsKey(it.plainName) } - .collectEntries { [ it.plainName, params[it.plainName] ] } - def globalID = params.get("id", null) - - /* process params_list arguments */ - /*********************************/ - def paramList = params.containsKey("param_list") && params.param_list != null ? - params.param_list : [] - // if (paramList instanceof String) { - // paramList = [paramList] - // } - // def paramSets = paramList.collectMany{ _parseParamList(it, config) } - // TODO: be able to process param_list when it is a list of strings - def paramSets = _parseParamList(paramList, config) - if (paramSets.isEmpty()) { - paramSets = [[null, [:]]] - } - - /* combine arguments into channel */ - /**********************************/ - def processedParams = paramSets.indexed().collect{ index, tup -> - // Process ID - def id = tup[0] ?: globalID - - if (workflow.stubRun && !id) { - // if stub run, explicitly add an id if missing - id = "stub${index}" - } - assert id != null: "Each parameter set should have at least an 'id'" - - // Process params - def parValues = globalParams + tup[1] - // // Remove parameters which are null, if the default is also null - // parValues = parValues.collectEntries{paramName, paramValue -> - // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) - // if ( paramValue != null || parameterSettings.get("default", null) != null ) { - // [paramName, paramValue] - // } - // } - parValues = parValues.collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" - - if (par == null) { - return [:] - } - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - - [ name, value ] - } - - [id, parValues] - } - - // Check if ids (first element of each list) is unique - _checkUniqueIds(processedParams) - return processedParams -} - -/** - * Parse nextflow parameters based on settings defined in a viash config - * and return a nextflow channel. - * - * @param params Input parameters. Can optionaly contain a 'param_list' key that - * provides a list of arguments that can be split up into multiple events - * in the output channel possible formats of param_lists are: a csv file, - * json file, a yaml file or a yaml blob. Each parameters set (event) must - * have a unique ID. - * @param config A Map of the Viash configuration. This Map can be generated from the config file - * using the readConfig() function. - * - * @return A nextflow Channel with events. Events are formatted as a tuple that contains - * first contains the ID of the event and as second element holds a parameter map. - * - * - */ -def channelFromParams(Map params, Map config) { - def processedParams = _paramsToParamSets(params, config) - return Channel.fromList(processedParams) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' -def checkUniqueIds(Map args) { - def stopOnError = args.stopOnError == null ? args.stopOnError : true - - def idChecker = new IDChecker() - - return filter { tup -> - if (!idChecker.observe(tup[0])) { - if (stopOnError) { - error "Duplicate id: ${tup[0]}" - } else { - log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" - return false - } - } - return true - } -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' -// This helper file will be deprecated soon -preprocessInputsDeprecationWarningPrinted = false - -def preprocessInputsDeprecationWarning() { - if (!preprocessInputsDeprecationWarningPrinted) { - preprocessInputsDeprecationWarningPrinted = true - System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") - } -} - -/** - * Generate a nextflow Workflow that allows processing a channel of - * Vdsl3 formatted events and apply a Viash config to them: - * - Gather default parameters from the Viash config and make - * sure that they are correctly formatted (see applyConfig method). - * - Format the input parameters (also using the applyConfig method). - * - Apply the default parameter to the input parameters. - * - Do some assertions: - * ~ Check if the event IDs in the channel are unique. - * - * The events in the channel are formatted as tuples, with the - * first element of the tuples being a unique id of the parameter set, - * and the second element containg the the parameters themselves. - * Optional extra elements of the tuples will be passed to the output as is. - * - * @param args A map that must contain a 'config' key that points - * to a parsed config (see readConfig()). Optionally, a - * 'key' key can be provided which can be used to create a unique - * name for the workflow process. - * - * @return A workflow that allows processing a channel of Vdsl3 formatted events - * and apply a Viash config to them. - */ -def preprocessInputs(Map args) { - preprocessInputsDeprecationWarning() - - def config = args.config - assert config instanceof Map : - "Error in preprocessInputs: config must be a map. " + - "Expected class: Map. Found: config.getClass() is ${config.getClass()}" - def key_ = args.key ?: config.name - - // Get different parameter types (used throughout this function) - def defaultArgs = config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - map { tup -> - def id = tup[0] - def data = tup[1] - def passthrough = tup.drop(2) - - def new_data = (defaultArgs + data).collectEntries { name, value -> - def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } - - if (par != null) { - value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") - } - - [ name, value ] - } - - [ id, new_data ] + passthrough - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component config. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component config. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component config. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component config. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runComponents(Map args) { - log.warn("runComponents is deprecated, use runEach instead") - assert args.components: "runComponents should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runComponents" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def id_ = args.id - - workflow runComponentsWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def comp_config = comp_.config - - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_config) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - // def new_id = id_(tup[0], tup[1], comp_config) - def new_id = tup[0] - if (id_ instanceof String) { - new_id = id_ - } else if (id_ instanceof Closure) { - new_id = id_(new_id, tup[1], comp_config) - } - [new_id] + tup.drop(1) - } - : filter_ch - def data_ch = id_ch | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_config) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_config) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - post_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runComponentsWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' -/** - * Run a list of components on a stream of data. - * - * @param components: list of Viash VDSL3 modules to run - * @param fromState: a closure, a map or a list of keys to extract from the input data. - * If a closure, it will be called with the id, the data and the component itself. - * @param toState: a closure, a map or a list of keys to extract from the output data - * If a closure, it will be called with the id, the output data, the old state and the component itself. - * @param filter: filter function to apply to the input. - * It will be called with the id, the data and the component itself. - * @param id: id to use for the output data - * If a closure, it will be called with the id, the data and the component itself. - * @param auto: auto options to pass to the components - * - * @return: a workflow that runs the components - **/ -def runEach(Map args) { - assert args.components: "runEach should be passed a list of components to run" - - def components_ = args.components - if (components_ !instanceof List) { - components_ = [ components_ ] - } - assert components_.size() > 0: "pass at least one component to runEach" - - def fromState_ = args.fromState - def toState_ = args.toState - def filter_ = args.filter - def runIf_ = args.runIf - def id_ = args.id - - assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." - - workflow runEachWf { - take: input_ch - main: - - // generate one channel per method - out_chs = components_.collect{ comp_ -> - def filter_ch = filter_ - ? input_ch | filter{tup -> - filter_(tup[0], tup[1], comp_) - } - : input_ch - def id_ch = id_ - ? filter_ch | map{tup -> - def new_id = id_ - if (new_id instanceof Closure) { - new_id = new_id(tup[0], tup[1], comp_) - } - assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" - [new_id] + tup.drop(1) - } - : filter_ch - def chPassthrough = null - def chRun = null - if (runIf_) { - def idRunIfBranch = id_ch.branch{ tup -> - run: runIf_(tup[0], tup[1], comp_) - passthrough: true - } - chPassthrough = idRunIfBranch.passthrough - chRun = idRunIfBranch.run - } else { - chRun = id_ch - chPassthrough = Channel.empty() - } - def data_ch = chRun | map{tup -> - def new_data = tup[1] - if (fromState_ instanceof Map) { - new_data = fromState_.collectEntries{ key0, key1 -> - [key0, new_data[key1]] - } - } else if (fromState_ instanceof List) { - new_data = fromState_.collectEntries{ key -> - [key, new_data[key]] - } - } else if (fromState_ instanceof Closure) { - new_data = fromState_(tup[0], new_data, comp_) - } - tup.take(1) + [new_data] + tup.drop(1) - } - def out_ch = data_ch - | comp_.run( - auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] - ) - def post_ch = toState_ - ? out_ch | map{tup -> - def output = tup[1] - def old_state = tup[2] - def new_state = null - if (toState_ instanceof Map) { - new_state = old_state + toState_.collectEntries{ key0, key1 -> - [key0, output[key1]] - } - } else if (toState_ instanceof List) { - new_state = old_state + toState_.collectEntries{ key -> - [key, output[key]] - } - } else if (toState_ instanceof Closure) { - new_state = toState_(tup[0], output, old_state, comp_) - } - [tup[0], new_state] + tup.drop(3) - } - : out_ch - - def return_ch = post_ch - | concat(chPassthrough) - - return_ch - } - - // mix all results - output_ch = - (out_chs.size == 1) - ? out_chs[0] - : out_chs[0].mix(*out_chs.drop(1)) - - emit: output_ch - } - - return runEachWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' -/** - * Join sourceChannel to targetChannel - * - * This function joins the sourceChannel to the targetChannel. - * However, each id in the targetChannel must be present in the - * sourceChannel. If _meta.join_id exists in the targetChannel, that is - * used as an id instead. If the id doesn't match any id in the sourceChannel, - * an error is thrown. - */ - -def safeJoin(targetChannel, sourceChannel, key) { - def sourceIDs = new IDChecker() - - def sourceCheck = sourceChannel - | map { tup -> - sourceIDs.observe(tup[0]) - tup - } - def targetCheck = targetChannel - | map { tup -> - def id = tup[0] - - if (!sourceIDs.contains(id)) { - error ( - "Error in module '${key}' when merging output with original state.\n" + - " Reason: output with id '${id}' could not be joined with source channel.\n" + - " If the IDs in the output channel differ from the input channel,\n" + - " please set `tup[1]._meta.join_id to the original ID.\n" + - " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + - " Unexpected ID in the output channel: '${id}'.\n" + - " Example input event: [\"id\", [input: file(...)]],\n" + - " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" - ) - } - // TODO: add link to our documentation on how to fix this - - tup - } - - sourceCheck.cross(targetChannel) - | map{ left, right -> - right + left.drop(1) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' -def _processArgument(arg) { - arg.multiple = arg.multiple != null ? arg.multiple : false - arg.required = arg.required != null ? arg.required : false - arg.direction = arg.direction != null ? arg.direction : "input" - arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" - arg.plainName = arg.name.replaceAll("^-*", "") - - if (arg.type == "file") { - arg.must_exist = arg.must_exist != null ? arg.must_exist : true - arg.create_parent = arg.create_parent != null ? arg.create_parent : true - } - - // add default values to output files which haven't already got a default - if (arg.type == "file" && arg.direction == "output" && arg.default == null) { - def mult = arg.multiple ? "_*" : "" - def extSearch = "" - if (arg.default != null) { - extSearch = arg.default - } else if (arg.example != null) { - extSearch = arg.example - } - if (extSearch instanceof List) { - extSearch = extSearch[0] - } - def extSearchResult = extSearch.find("\\.[^\\.]+\$") - def ext = extSearchResult != null ? extSearchResult : "" - arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" - if (arg.multiple) { - arg.default = [arg.default] - } - } - - if (!arg.multiple) { - if (arg.default != null && arg.default instanceof List) { - arg.default = arg.default[0] - } - if (arg.example != null && arg.example instanceof List) { - arg.example = arg.example[0] - } - } - - if (arg.type == "boolean_true") { - arg.default = false - } - if (arg.type == "boolean_false") { - arg.default = true - } - - arg -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' -def addGlobalArguments(config) { - def localConfig = [ - "argument_groups": [ - [ - "name": "Nextflow input-output arguments", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "arguments" : [ - [ - 'name': '--publish_dir', - 'required': true, - 'type': 'string', - 'description': 'Path to an output directory.', - 'example': 'output/', - 'multiple': false - ], - [ - 'name': '--param_list', - 'required': false, - 'type': 'string', - 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. - | - |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. - |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. - |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. - |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. - | - |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), - 'example': 'my_params.yaml', - 'multiple': false, - 'hidden': true - ] - // TODO: allow multiple: true in param_list? - // TODO: allow to specify a --param_list_regex to filter the param_list? - // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? - ] - ] - ] - ] - - return processConfig(_mergeMap(config, localConfig)) -} - -def _mergeMap(Map lhs, Map rhs) { - return rhs.inject(lhs.clone()) { map, entry -> - if (map[entry.key] instanceof Map && entry.value instanceof Map) { - map[entry.key] = _mergeMap(map[entry.key], entry.value) - } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { - map[entry.key] += entry.value - } else { - map[entry.key] = entry.value - } - return map - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' -def _generateArgumentHelp(param) { - // alternatives are not supported - // def names = param.alternatives ::: List(param.name) - - def unnamedProps = [ - ["required parameter", param.required], - ["multiple values allowed", param.multiple], - ["output", param.direction.toLowerCase() == "output"], - ["file must exist", param.type == "file" && param.must_exist] - ].findAll{it[1]}.collect{it[0]} - - def dflt = null - if (param.default != null) { - if (param.default instanceof List) { - dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - dflt = param.default.toString() - } - } - def example = null - if (param.example != null) { - if (param.example instanceof List) { - example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") - } else { - example = param.example.toString() - } - } - def min = param.min?.toString() - def max = param.max?.toString() - - def escapeChoice = { choice -> - def s1 = choice.replaceAll("\\n", "\\\\n") - def s2 = s1.replaceAll("\"", """\\\"""") - s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 - } - def choices = param.choices == null ? - null : - "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" - - def namedPropsStr = [ - ["type", ([param.type] + unnamedProps).join(", ")], - ["default", dflt], - ["example", example], - ["choices", choices], - ["min", min], - ["max", max] - ] - .findAll{it[1]} - .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} - .join("") - - def descStr = param.description == null ? - "" : - _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") - - "\n --" + param.plainName + - namedPropsStr + - descStr -} - -// Based on Helper.generateHelp() in Helper.scala -def _generateHelp(config) { - def fun = config - - // PART 1: NAME AND VERSION - def nameStr = fun.name + - (fun.version == null ? "" : " " + fun.version) - - // PART 2: DESCRIPTION - def descrStr = fun.description == null ? - "" : - "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") - - // PART 3: Usage - def usageStr = fun.usage == null ? - "" : - "\n\nUsage:\n" + fun.usage.trim() - - // PART 4: Options - def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> - def name = argGroup.name - def descriptionStr = argGroup.description == null ? - "" : - "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" - def arguments = argGroup.arguments.collect{arg -> - arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg - }.findAll{it != null} - def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} - - "\n\n$name:" + - descriptionStr + - argumentStrs.join("\n") - } - - // FINAL: combine - def out = nameStr + - descrStr + - usageStr + - argGroupStrs.join("") - - return out -} - -// based on Format._paragraphWrap -def _paragraphWrap(str, maxLength) { - def outLines = [] - str.split("\n").each{par -> - def words = par.split("\\s").toList() - - def word = null - def line = words.pop() - while(!words.isEmpty()) { - word = words.pop() - if (line.length() + word.length() + 1 <= maxLength) { - line = line + " " + word - } else { - outLines.add(line) - line = word - } - } - if (words.isEmpty()) { - outLines.add(line) - } - } - return outLines -} - -def helpMessage(config) { - if (params.containsKey("help") && params.help) { - def mergedConfig = addGlobalArguments(config) - def helpStr = _generateHelp(mergedConfig) - println(helpStr) - exit 0 - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' -def processConfig(config) { - // set defaults for arguments - config.arguments = - (config.arguments ?: []).collect{_processArgument(it)} - - // set defaults for argument_group arguments - config.argument_groups = - (config.argument_groups ?: []).collect{grp -> - grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} - grp - } - - // create combined arguments list - config.allArguments = - config.arguments + - config.argument_groups.collectMany{it.arguments} - - // add missing argument groups (based on Functionality::allArgumentGroups()) - def argGroups = config.argument_groups - if (argGroups.any{it.name.toLowerCase() == "arguments"}) { - argGroups = argGroups.collect{ grp -> - if (grp.name.toLowerCase() == "arguments") { - grp = grp + [ - arguments: grp.arguments + config.arguments - ] - } - grp - } - } else { - argGroups = argGroups + [ - name: "Arguments", - arguments: config.arguments - ] - } - config.allArgumentGroups = argGroups - - config -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' - -def readConfig(file) { - def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) - processConfig(config) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' -/** - * Resolve a path relative to the current file. - * - * @param str The path to resolve, as a String. - * @param parentPath The path to resolve relative to, as a Path. - * - * @return The path that may have been resovled, as a Path. - */ -def _resolveSiblingIfNotAbsolute(str, parentPath) { - if (str !instanceof String) { - return str - } - if (!_stringIsAbsolutePath(str)) { - return parentPath.resolveSibling(str) - } else { - return file(str, hidden: true) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' -/** - * Check whether a path as a string is absolute. - * - * In the past, we tried using `file(., relative: true).isAbsolute()`, - * but the 'relative' option was added in 22.10.0. - * - * @param path The path to check, as a String. - * - * @return Whether the path is absolute, as a boolean. - */ -def _stringIsAbsolutePath(path) { - def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ - - assert path instanceof String - return _resolve_URL_PROTOCOL.matcher(path).matches() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' -class CustomTraceObserver implements nextflow.trace.TraceObserver { - List traces - - CustomTraceObserver(List traces) { - this.traces = traces - } - - @Override - void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } - - @Override - void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { - def trace2 = trace.store.clone() - trace2.script = null - traces.add(trace2) - } -} - -def collectTraces() { - def traces = Collections.synchronizedList([]) - - // add custom trace observer which stores traces in the traces object - session.observers.add(new CustomTraceObserver(traces)) - - traces -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' -/** - * Performs a deep clone of the given object. - * @param x an object - */ -def deepClone(x) { - iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' -def getPublishDir() { - return params.containsKey("publish_dir") ? params.publish_dir : - params.containsKey("publishDir") ? params.publishDir : - null -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' - -// Recurse upwards until we find a '.build.yaml' file -def _findBuildYamlFile(pathPossiblySymlink) { - def path = pathPossiblySymlink.toRealPath() - def child = path.resolve(".build.yaml") - if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { - return child - } else { - def parent = path.getParent() - if (parent == null) { - return null - } else { - return _findBuildYamlFile(parent) - } - } -} - -// get the root of the target folder -def getRootDir() { - def dir = _findBuildYamlFile(meta.resources_dir) - assert dir != null: "Could not find .build.yaml in the folder structure" - dir.getParent() -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' -/** - * Recursively apply a function over the leaves of an object. - * @param obj The object to iterate over. - * @param fun The function to apply to each value. - * @return The object with the function applied to each value. - */ -def iterateMap(obj, fun) { - if (obj instanceof List && obj !instanceof String) { - return obj.collect{item -> - iterateMap(item, fun) - } - } else if (obj instanceof Map) { - return obj.collectEntries{key, item -> - [key.toString(), iterateMap(item, fun)] - } - } else { - return fun(obj) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' -/** - * A view for printing the event of each channel as a YAML blob. - * This is useful for debugging. - */ -def niceView() { - workflow niceViewWf { - take: input - main: - output = input - | view{toYamlBlob(it)} - emit: output - } - return niceViewWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' - -def readCsv(file_path) { - def output = [] - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - - // todo: allow escaped quotes in string - // todo: allow single quotes? - def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') - def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') - - def br = java.nio.file.Files.newBufferedReader(inputFile) - - def row = -1 - def header = null - while (br.ready() && header == null) { - def line = br.readLine() - row++ - if (!line.startsWith("#")) { - header = splitRegex.split(line, -1).collect{field -> - m = removeQuote.matcher(field) - m.find() ? m.replaceFirst('$1') : field - } - } - } - assert header != null: "CSV file should contain a header" - - while (br.ready()) { - def line = br.readLine() - row++ - if (line == null) { - br.close() - break - } - - if (!line.startsWith("#")) { - def predata = splitRegex.split(line, -1) - def data = predata.collect{field -> - if (field == "") { - return null - } - def m = removeQuote.matcher(field) - if (m.find()) { - return m.replaceFirst('$1') - } else { - return field - } - } - assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" - - def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} - output.add(dataMap) - } - } - - output -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' -def readJson(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parse(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' -def readJsonBlob(str) { - def jsonSlurper = new groovy.json.JsonSlurper() - jsonSlurper.parseText(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' -// Custom constructor to modify how certain objects are parsed from YAML -class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { - Path root - - class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { - public Object construct(org.yaml.snakeyaml.nodes.Node node) { - String filename = (String) constructScalar(node); - if (root != null) { - return root.resolve(filename); - } - return java.nio.file.Paths.get(filename); - } - } - - CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { - super(options) - this.root = root - // Handling !file tag and parse it back to a File type - this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) - } -} - -def readTaggedYaml(Path path) { - def options = new org.yaml.snakeyaml.LoaderOptions() - def constructor = new CustomConstructor(options, path.getParent()) - def yaml = new org.yaml.snakeyaml.Yaml(constructor) - return yaml.load(path.text) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' -def readYaml(file_path) { - def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(inputFile) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' -def readYamlBlob(str) { - def yamlSlurper = new org.yaml.snakeyaml.Yaml() - yamlSlurper.load(str) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' -String toJsonBlob(data) { - return groovy.json.JsonOutput.toJson(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' -// Custom representer to modify how certain objects are represented in YAML -class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { - Path relativizer - - class RepresentPath implements org.yaml.snakeyaml.representer.Represent { - public String getFileName(Object obj) { - if (obj instanceof File) { - obj = ((File) obj).toPath(); - } - if (obj !instanceof Path) { - throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); - } - def path = (Path) obj; - - if (relativizer != null) { - return relativizer.relativize(path).toString() - } else { - return path.toString() - } - } - - public org.yaml.snakeyaml.nodes.Node representData(Object data) { - String filename = getFileName(data); - def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); - return representScalar(tag, filename); - } - } - CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { - super(options) - this.relativizer = relativizer - this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) - this.representers.put(Path, new RepresentPath()) - this.representers.put(File, new RepresentPath()) - } -} - -String toTaggedYamlBlob(data) { - return toRelativeTaggedYamlBlob(data, null) -} -String toRelativeTaggedYamlBlob(data, Path relativizer) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - def representer = new CustomRepresenter(options, relativizer) - def yaml = new org.yaml.snakeyaml.Yaml(representer, options) - return yaml.dump(data) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' -String toYamlBlob(data) { - def options = new org.yaml.snakeyaml.DumperOptions() - options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) - options.setPrettyFlow(true) - def yaml = new org.yaml.snakeyaml.Yaml(options) - def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) - return yaml.dump(cleanData) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' -void writeJson(data, file) { - assert data: "writeJson: data should not be null" - assert file: "writeJson: file should not be null" - file.write(toJsonBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' -void writeYaml(data, file) { - assert data: "writeYaml: data should not be null" - assert file: "writeYaml: file should not be null" - file.write(toYamlBlob(data)) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' -def findStates(Map params, Map config) { - def auto_config = deepClone(config) - def auto_params = deepClone(params) - - auto_config = auto_config.clone() - // override arguments - auto_config.argument_groups = [] - auto_config.arguments = [ - [ - type: "string", - name: "--id", - description: "A dummy identifier", - required: false - ], - [ - type: "file", - name: "--input_states", - example: "/path/to/input/directory/**/state.yaml", - description: "Path to input directory containing the datasets to be integrated.", - required: true, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--filter", - example: "foo/.*/state.yaml", - description: "Regex to filter state files by path.", - required: false - ], - // to do: make this a yaml blob? - [ - type: "string", - name: "--rename_keys", - example: ["newKey1:oldKey1", "newKey2:oldKey2"], - description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", - required: false, - multiple: true, - multiple_sep: ";" - ], - [ - type: "string", - name: "--settings", - example: '{"output_dataset": "dataset.h5ad", "k": 10}', - description: "Global arguments as a JSON glob to be passed to all components.", - required: false - ] - ] - if (!(auto_params.containsKey("id"))) { - auto_params["id"] = "auto" - } - - // run auto config through processConfig once more - auto_config = processConfig(auto_config) - - workflow findStatesWf { - helpMessage(auto_config) - - output_ch = - channelFromParams(auto_params, auto_config) - | flatMap { autoId, args -> - - def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] - - // look for state files in input dir - def stateFiles = args.input_states - - // filter state files by regex - if (args.filter) { - stateFiles = stateFiles.findAll{ stateFile -> - def stateFileStr = stateFile.toString() - def matcher = stateFileStr =~ args.filter - matcher.matches()} - } - - // read in states - def states = stateFiles.collect { stateFile -> - def state_ = readTaggedYaml(stateFile) - [state_.id, state_] - } - - // construct renameMap - if (args.rename_keys) { - def renameMap = args.rename_keys.collectEntries{renameString -> - def split = renameString.split(":") - assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" - split - } - - // rename keys in state, only let states through which have all keys - // also add global settings - states = states.collectMany{id, state -> - def newState = [:] - - for (key in renameMap.keySet()) { - def origKey = renameMap[key] - if (!(state.containsKey(origKey))) { - return [] - } - newState[key] = state[origKey] - } - - [[id, globalSettings + newState]] - } - } - - states - } - emit: - output_ch - } - - return findStatesWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' -def joinStates(Closure apply_) { - workflow joinStatesWf { - take: input_ch - main: - output_ch = input_ch - | toSortedList - | filter{ it.size() > 0 } - | map{ tups -> - def ids = tups.collect{it[0]} - def states = tups.collect{it[1]} - apply_(ids, states) - } - - emit: output_ch - } - return joinStatesWf -} -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' -def publishFiles(Map args) { - def key_ = args.get("key") - - assert key_ != null : "publishFiles: key must be specified" - - workflow publishFilesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - def inputFiles_ = inputoutputFilenames_[0] - def outputFilenames_ = inputoutputFilenames_[1] - - [id_, inputFiles_, outputFilenames_] - } - | publishFilesProc - emit: input_ch - } - return publishFilesWf -} - -process publishFilesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) - output: - tuple val(id), path{outputFiles} - script: - def copyCommands = [ - inputFiles instanceof List ? inputFiles : [inputFiles], - outputFiles instanceof List ? outputFiles : [outputFiles] - ] - .transpose() - .collectMany{infile, outfile -> - if (infile.toString() != outfile.toString()) { - [ - "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", - "cp -r '${infile.toString()}' '${outfile.toString()}'" - ] - } else { - // no need to copy if infile is the same as outfile - [] - } - } - """ - echo "Copying output files to destination folder" - ${copyCommands.join("\n ")} - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishFilesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishFilesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishFilesByConfig: key must be specified" - - workflow publishFilesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - - // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - inputPath is a List[Path] - // - outputFilename is a List[String] - // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output OR multiple channels were emitted - // and the output was just not added to using the channel - // that is now being parsed - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[inputPath: [], outputFilename: []]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def inputPath = val instanceof File ? val.toPath() : val - [inputPath: inputPath, outputFilename: filename_ix] - } - def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> - [key, outputPerFile.collect{dic -> dic[key]}] - } - return [[key: plainName_] + transposedOutputs] - } else { - def value_ = java.nio.file.Paths.get(filename) - def inputPath = value instanceof File ? value.toPath() : value - return [[inputPath: [inputPath], outputFilename: [filename]]] - } - } - - def inputPaths = processedState.collectMany{it.inputPath} - def outputFilenames = processedState.collectMany{it.outputFilename} - - - [id_, inputPaths, outputFilenames] - } - | publishFilesProc - emit: input_ch - } - return publishFilesSimpleWf -} - - - - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' -def collectFiles(obj) { - if (obj instanceof java.io.File || obj instanceof Path) { - return [obj] - } else if (obj instanceof List && obj !instanceof String) { - return obj.collectMany{item -> - collectFiles(item) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectFiles(item) - } - } else { - return [] - } -} - -/** - * Recurse through a state and collect all input files and their target output filenames. - * @param obj The state to recurse through. - * @param prefix The prefix to prepend to the output filenames. - */ -def collectInputOutputPaths(obj, prefix) { - if (obj instanceof File || obj instanceof Path) { - def path = obj instanceof Path ? obj : obj.toPath() - def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" - def newFilename = prefix + ext - return [[obj, newFilename]] - } else if (obj instanceof List && obj !instanceof String) { - return obj.withIndex().collectMany{item, ix -> - collectInputOutputPaths(item, prefix + "_" + ix) - } - } else if (obj instanceof Map) { - return obj.collectMany{key, item -> - collectInputOutputPaths(item, prefix + "." + key) - } - } else { - return [] - } -} - -def publishStates(Map args) { - def key_ = args.get("key") - def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) - - assert key_ != null : "publishStates: key must be specified" - - workflow publishStatesWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] - - // the input files and the target output filenames - def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() - - def yamlFilename = yamlTemplate_ - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - - // TODO: do the pathnames in state_ match up with the outputFilenames_? - - // convert state to yaml blob - def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesWf -} -process publishStatesProc { - // todo: check publishpath? - publishDir path: "${getPublishDir()}/", mode: "copy" - tag "$id" - input: - tuple val(id), val(yamlBlob), val(yamlFile) - output: - tuple val(id), path{[yamlFile]} - script: - """ - mkdir -p "\$(dirname '${yamlFile}')" - echo "Storing state as yaml" - cat > '${yamlFile}' << HERE -${yamlBlob} -HERE - """ -} - - -// this assumes that the state contains no other values other than those specified in the config -def publishStatesByConfig(Map args) { - def config = args.get("config") - assert config != null : "publishStatesByConfig: config must be specified" - - def key_ = args.get("key", config.name) - assert key_ != null : "publishStatesByConfig: key must be specified" - - workflow publishStatesSimpleWf { - take: input_ch - main: - input_ch - | map { tup -> - def id_ = tup[0] - def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] - def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] - - // TODO: allow overriding the state.yaml template - // TODO TODO: if auto.publish == "state", add output_state as an argument - def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' - def yamlFilename = yamlTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() - - // the processed state is a list of [key, value] tuples, where - // - key is a String - // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) - // - (key, value) are the tuples that will be saved to the state.yaml file - def processedState = - config.allArguments - .findAll { it.direction == "output" } - .collectMany { par -> - def plainName_ = par.plainName - // if the state does not contain the key, it's an - // optional argument for which the component did - // not generate any output - if (!state_.containsKey(plainName_)) { - return [] - } - def value = state_[plainName_] - // if the parameter is not a file, it should be stored - // in the state as-is, but is not something that needs - // to be copied from the source path to the dest path - if (par.type != "file") { - return [[key: plainName_, value: value]] - } - // if the orig state does not contain this filename, - // it's an optional argument for which the user specified - // that it should not be returned as a state - if (!origState_.containsKey(plainName_)) { - return [] - } - def filenameTemplate = origState_[plainName_] - // if the pararameter is multiple: true, fetch the template - if (par.multiple && filenameTemplate instanceof List) { - filenameTemplate = filenameTemplate[0] - } - // instantiate the template - def filename = filenameTemplate - .replaceAll('\\$id', id_) - .replaceAll('\\$\\{id\\}', id_) - .replaceAll('\\$key', key_) - .replaceAll('\\$\\{key\\}', key_) - if (par.multiple) { - // if the parameter is multiple: true, the filename - // should contain a wildcard '*' that is replaced with - // the index of the file - assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" - def outputPerFile = value.withIndex().collect{ val, ix -> - def filename_ix = filename.replace("*", ix.toString()) - def value_ = java.nio.file.Paths.get(filename_ix) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - return value_ - } - return [["key": plainName_, "value": outputPerFile]] - } else { - def value_ = java.nio.file.Paths.get(filename) - // if id contains a slash - if (yamlDir != null) { - value_ = yamlDir.relativize(value_) - } - def inputPath = value instanceof File ? value.toPath() : value - return [["key": plainName_, value: value_]] - } - } - - - def updatedState_ = processedState.collectEntries{[it.key, it.value]} - - // convert state to yaml blob - def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) - - [id_, yamlBlob_, yamlFilename] - } - | publishStatesProc - emit: input_ch - } - return publishStatesSimpleWf -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' -def setState(fun) { - assert fun instanceof Closure || fun instanceof Map || fun instanceof List : - "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" - - // if fun is a List, convert to map - if (fun instanceof List) { - // check whether fun is a list[string] - assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" - fun = fun.collectEntries{[it, it]} - } - - // if fun is a map, convert to closure - if (fun instanceof Map) { - // check whether fun is a map[string, string] - assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" - assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" - def funMap = fun.clone() - // turn the map into a closure to be used later on - fun = { id_, state_ -> - assert state_ instanceof Map : "Error in setState: the state is not a Map" - funMap.collectMany{newkey, origkey -> - if (state_.containsKey(origkey)) { - [[newkey, state_[origkey]]] - } else { - [] - } - }.collectEntries() - } - } - - map { tup -> - def id = tup[0] - def state = tup[1] - def unfilteredState = fun(id, state) - def newState = unfilteredState.findAll{key, val -> val != null} - [id, newState] + tup.drop(2) - } -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' -// TODO: unit test processAuto -def processAuto(Map auto) { - // remove null values - auto = auto.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] - def unexpectedKeys = auto.keySet() - expectedKeys - assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" - - // check auto.simplifyInput - assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" - - // check auto.simplifyOutput - assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" - - // check auto.transcript - assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" - - // check auto.publish - assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" - - return auto.subMap(expectedKeys) -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' -def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { - assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" - map.forEach { key, val -> - assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" - } - requiredKeys.forEach { requiredKey -> - assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" - } -} - -// TODO: unit test processDirectives -def processDirectives(Map drctv) { - // remove null values - drctv = drctv.findAll{k, v -> v != null} - - // check for unexpected keys - def expectedKeys = [ - "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" - ] - def unexpectedKeys = drctv.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" - - /* DIRECTIVE accelerator - accepted examples: - - [ limit: 4, type: "nvidia-tesla-k80" ] - */ - if (drctv.containsKey("accelerator")) { - assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") - } - - /* DIRECTIVE afterScript - accepted examples: - - "source /cluster/bin/cleanup" - */ - if (drctv.containsKey("afterScript")) { - assert drctv["afterScript"] instanceof CharSequence - } - - /* DIRECTIVE beforeScript - accepted examples: - - "source /cluster/bin/setup" - */ - if (drctv.containsKey("beforeScript")) { - assert drctv["beforeScript"] instanceof CharSequence - } - - /* DIRECTIVE cache - accepted examples: - - true - - false - - "deep" - - "lenient" - */ - if (drctv.containsKey("cache")) { - assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean - if (drctv["cache"] instanceof CharSequence) { - assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" - } - } - - /* DIRECTIVE conda - accepted examples: - - "bwa=0.7.15" - - "bwa=0.7.15 fastqc=0.11.5" - - ["bwa=0.7.15", "fastqc=0.11.5"] - */ - if (drctv.containsKey("conda")) { - if (drctv["conda"] instanceof List) { - drctv["conda"] = drctv["conda"].join(" ") - } - assert drctv["conda"] instanceof CharSequence - } - - /* DIRECTIVE container - accepted examples: - - "foo/bar:tag" - - [ registry: "reg", image: "im", tag: "ta" ] - is transformed to "reg/im:ta" - - [ image: "im" ] - is transformed to "im:latest" - */ - if (drctv.containsKey("container")) { - assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence - if (drctv["container"] instanceof Map) { - def m = drctv["container"] - assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") - def part1 = - System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : - params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? - m.registry ? m.registry + "/" : - "" - def part2 = m.image - def part3 = m.tag ? ":" + m.tag : ":latest" - drctv["container"] = part1 + part2 + part3 - } - } - - /* DIRECTIVE containerOptions - accepted examples: - - "--foo bar" - - ["--foo bar", "-f b"] - */ - if (drctv.containsKey("containerOptions")) { - if (drctv["containerOptions"] instanceof List) { - drctv["containerOptions"] = drctv["containerOptions"].join(" ") - } - assert drctv["containerOptions"] instanceof CharSequence - } - - /* DIRECTIVE cpus - accepted examples: - - 1 - - 10 - */ - if (drctv.containsKey("cpus")) { - assert drctv["cpus"] instanceof Integer - } - - /* DIRECTIVE disk - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("disk")) { - assert drctv["disk"] instanceof CharSequence - // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE echo - accepted examples: - - true - - false - */ - if (drctv.containsKey("echo")) { - assert drctv["echo"] instanceof Boolean - } - - /* DIRECTIVE errorStrategy - accepted examples: - - "terminate" - - "finish" - */ - if (drctv.containsKey("errorStrategy")) { - assert drctv["errorStrategy"] instanceof CharSequence - assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" - } - - /* DIRECTIVE executor - accepted examples: - - "local" - - "sge" - */ - if (drctv.containsKey("executor")) { - assert drctv["executor"] instanceof CharSequence - assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" - } - - /* DIRECTIVE machineType - accepted examples: - - "n1-highmem-8" - */ - if (drctv.containsKey("machineType")) { - assert drctv["machineType"] instanceof CharSequence - } - - /* DIRECTIVE maxErrors - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxErrors")) { - assert drctv["maxErrors"] instanceof Integer - } - - /* DIRECTIVE maxForks - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxForks")) { - assert drctv["maxForks"] instanceof Integer - } - - /* DIRECTIVE maxRetries - accepted examples: - - 1 - - 3 - */ - if (drctv.containsKey("maxRetries")) { - assert drctv["maxRetries"] instanceof Integer - } - - /* DIRECTIVE memory - accepted examples: - - "1 GB" - - "2TB" - - "3.2KB" - - "10.B" - */ - if (drctv.containsKey("memory")) { - assert drctv["memory"] instanceof CharSequence - // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") - // ^ does not allow closures - } - - /* DIRECTIVE module - accepted examples: - - "ncbi-blast/2.2.27" - - "ncbi-blast/2.2.27:t_coffee/10.0" - - ["ncbi-blast/2.2.27", "t_coffee/10.0"] - */ - if (drctv.containsKey("module")) { - if (drctv["module"] instanceof List) { - drctv["module"] = drctv["module"].join(":") - } - assert drctv["module"] instanceof CharSequence - } - - /* DIRECTIVE penv - accepted examples: - - "smp" - */ - if (drctv.containsKey("penv")) { - assert drctv["penv"] instanceof CharSequence - } - - /* DIRECTIVE pod - accepted examples: - - [ label: "key", value: "val" ] - - [ annotation: "key", value: "val" ] - - [ env: "key", value: "val" ] - - [ [label: "l", value: "v"], [env: "e", value: "v"]] - */ - if (drctv.containsKey("pod")) { - if (drctv["pod"] instanceof Map) { - drctv["pod"] = [ drctv["pod"] ] - } - assert drctv["pod"] instanceof List - drctv["pod"].forEach { pod -> - assert pod instanceof Map - // TODO: should more checks be added? - // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod - // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? - } - } - - /* DIRECTIVE publishDir - accepted examples: - - [] - - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] - - "/path/to/dir" - is transformed to [[ path: "/path/to/dir" ]] - - [ path: "/path/to/dir", mode: "cache" ] - is transformed to [[ path: "/path/to/dir", mode: "cache" ]] - */ - // TODO: should we also look at params["publishDir"]? - if (drctv.containsKey("publishDir")) { - def pblsh = drctv["publishDir"] - - // check different options - assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence - - // turn into list if not already so - // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. - pblsh = pblsh instanceof List ? pblsh : [ pblsh ] - - // check elements of publishDir - pblsh = pblsh.collect{ elem -> - // turn into map if not already so - elem = elem instanceof CharSequence ? [ path: elem ] : elem - - // check types and keys - assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" - assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") - - // check elements in map - assert elem.containsKey("path") - assert elem["path"] instanceof CharSequence - if (elem.containsKey("mode")) { - assert elem["mode"] instanceof CharSequence - assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] - } - if (elem.containsKey("overwrite")) { - assert elem["overwrite"] instanceof Boolean - } - if (elem.containsKey("pattern")) { - assert elem["pattern"] instanceof CharSequence - } - if (elem.containsKey("saveAs")) { - assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" - } - if (elem.containsKey("enabled")) { - assert elem["enabled"] instanceof Boolean - } - - // return final result - elem - } - // store final directive - drctv["publishDir"] = pblsh - } - - /* DIRECTIVE queue - accepted examples: - - "long" - - "short,long" - - ["short", "long"] - */ - if (drctv.containsKey("queue")) { - if (drctv["queue"] instanceof List) { - drctv["queue"] = drctv["queue"].join(",") - } - assert drctv["queue"] instanceof CharSequence - } - - /* DIRECTIVE label - accepted examples: - - "big_mem" - - "big_cpu" - - ["big_mem", "big_cpu"] - */ - if (drctv.containsKey("label")) { - if (drctv["label"] instanceof CharSequence) { - drctv["label"] = [ drctv["label"] ] - } - assert drctv["label"] instanceof List - drctv["label"].forEach { label -> - assert label instanceof CharSequence - // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") - // ^ does not allow closures - } - } - - /* DIRECTIVE scratch - accepted examples: - - true - - "/path/to/scratch" - - '$MY_PATH_TO_SCRATCH' - - "ram-disk" - */ - if (drctv.containsKey("scratch")) { - assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence - } - - /* DIRECTIVE storeDir - accepted examples: - - "/path/to/storeDir" - */ - if (drctv.containsKey("storeDir")) { - assert drctv["storeDir"] instanceof CharSequence - } - - /* DIRECTIVE stageInMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageInMode")) { - assert drctv["stageInMode"] instanceof CharSequence - assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] - } - - /* DIRECTIVE stageOutMode - accepted examples: - - "copy" - - "link" - */ - if (drctv.containsKey("stageOutMode")) { - assert drctv["stageOutMode"] instanceof CharSequence - assert drctv["stageOutMode"] in ["copy", "move", "rsync"] - } - - /* DIRECTIVE tag - accepted examples: - - "foo" - - '$id' - */ - if (drctv.containsKey("tag")) { - assert drctv["tag"] instanceof CharSequence - } - - /* DIRECTIVE time - accepted examples: - - "1h" - - "2days" - - "1day 6hours 3minutes 30seconds" - */ - if (drctv.containsKey("time")) { - assert drctv["time"] instanceof CharSequence - // todo: validation regex? - } - - return drctv -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' -def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { - // override defaults with args - def workflowArgs = defaultWfArgs + args - - // check whether 'key' exists - assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" - - // if 'key' is a closure, apply it to the original key - if (workflowArgs["key"] instanceof Closure) { - workflowArgs["key"] = workflowArgs["key"](meta.config.name) - } - def key = workflowArgs["key"] - assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" - assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" - - // check for any unexpected keys - def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] - def unexpectedKeys = workflowArgs.keySet() - expectedKeys - assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" - assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" - workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) - - // check whether directives exists and apply defaults - assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" - assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" - workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) - - // auto define publish, if so desired - if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + - // " Example: params.publish_dir = \"./output/\"" - def publishDir = getPublishDir() - - if (publishDir != null) { - workflowArgs.directives.publishDir = [[ - path: publishDir, - saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default - mode: "copy" - ]] - } - } - - // auto define transcript, if so desired - if (workflowArgs.auto.transcript == true) { - // can't assert at this level thanks to the no_publish profile - // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : - // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + - // " Example: params.transcripts_dir = \"./transcripts/\"" - def transcriptsDir = - params.containsKey("transcripts_dir") ? params.transcripts_dir : - params.containsKey("transcriptsDir") ? params.transcriptsDir : - params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : - params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : - null - if (transcriptsDir != null) { - def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') - def transcriptsPublishDir = [ - path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", - saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", - mode: "copy" - ] - def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] - workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir - } - } - - // if this is a stubrun, remove certain directives? - if (workflow.stubRun) { - workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) - } - - for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { - assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" - } - } - - // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? - for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { - if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { - log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." - } - } - - // check fromState - workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) - - // check toState - workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) - - // return output - return workflowArgs -} - -def _processFromState(fromState, key_, config_) { - assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : - "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" - if (fromState == null) { - return null - } - - // if fromState is a List, convert to map - if (fromState instanceof List) { - // check whether fromstate is a list[string] - assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" - fromState = fromState.collectEntries{[it, it]} - } - - // if fromState is a map, convert to closure - if (fromState instanceof Map) { - // check whether fromstate is a map[string, string] - assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" - assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" - def fromStateMap = fromState.clone() - def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} - // turn the map into a closure to be used later on - fromState = { it -> - def state = it[1] - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def data = fromStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (state.containsKey(origkey)) { - [[newkey, state[origkey]]] - } else if (!requiredInputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") - } - }.collectEntries() - data - } - } - - return fromState -} - -def _processToState(toState, key_, config_) { - if (toState == null) { - toState = { tup -> tup[1] } - } - - // toState should be a closure, map[string, string], or list[string] - assert toState instanceof Closure || toState instanceof Map || toState instanceof List : - "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" - - // if toState is a List, convert to map - if (toState instanceof List) { - // check whether toState is a list[string] - assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" - toState = toState.collectEntries{[it, it]} - } - - // if toState is a map, convert to closure - if (toState instanceof Map) { - // check whether toState is a map[string, string] - assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" - assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" - def toStateMap = toState.clone() - def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} - // turn the map into a closure to be used later on - toState = { it -> - def output = it[1] - def state = it[2] - assert output instanceof Map : "Error in module '$key_': the output is not a Map" - assert state instanceof Map : "Error in module '$key_': the state is not a Map" - def extraEntries = toStateMap.collectMany{newkey, origkey -> - // check whether newkey corresponds to a required argument - if (output.containsKey(origkey)) { - [[newkey, output[origkey]]] - } else if (!requiredOutputNames.contains(origkey)) { - [] - } else { - throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") - } - }.collectEntries() - state + extraEntries - } - } - - return toState -} - -// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' -def _debug(workflowArgs, debugKey) { - if (workflowArgs.debug) { - view { "process '${workflowArgs.key}' $debugKey tuple: $it" } - } else { - map { it } - } -} - -// depends on: innerWorkflowFactory -def workflowFactory(Map args, Map defaultWfArgs, Map meta) { - def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) - def key_ = workflowArgs["key"] - def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} - - workflow workflowInstance { - take: input_ - - main: - def chModified = input_ - | checkUniqueIds([:]) - | _debug(workflowArgs, "input") - | map { tuple -> - tuple = deepClone(tuple) - - if (workflowArgs.map) { - tuple = workflowArgs.map(tuple) - } - if (workflowArgs.mapId) { - tuple[0] = workflowArgs.mapId(tuple[0]) - } - if (workflowArgs.mapData) { - tuple[1] = workflowArgs.mapData(tuple[1]) - } - if (workflowArgs.mapPassthrough) { - tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) - } - - // check tuple - assert tuple instanceof List : - "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - assert tuple.size() >= 2 : - "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: tuple.size() == ${tuple.size()}" - - // check id field - if (tuple[0] instanceof GString) { - tuple[0] = tuple[0].toString() - } - assert tuple[0] instanceof CharSequence : - "Error in module '${key_}': first element of tuple in channel should be a String\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Found: ${tuple[0]}" - - // match file to input file - if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { - def inputFiles = meta.config.allArguments - .findAll { it.type == "file" && it.direction == "input" } - - assert inputFiles.size() == 1 : - "Error in module '${key_}' id '${tuple[0]}'.\n" + - " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + - " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" - - tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() - } - - // check data field - assert tuple[1] instanceof Map : - "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // rename keys of data field in tuple - if (workflowArgs.renameKeys) { - assert workflowArgs.renameKeys instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" - assert tuple[1] instanceof Map : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" - - // TODO: allow renameKeys to be a function? - workflowArgs.renameKeys.each { newKey, oldKey -> - assert newKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" - assert oldKey instanceof CharSequence : - "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + - " Example: renameKeys: ['new_key': 'old_key'].\n" + - " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" - assert tuple[1].containsKey(oldKey) : - "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + - " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" - tuple[1].put(newKey, tuple[1][oldKey]) - } - tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) - } - tuple - } - - - def chRun = null - def chPassthrough = null - if (workflowArgs.runIf) { - def runIfBranch = chModified.branch{ tup -> - run: workflowArgs.runIf(tup[0], tup[1]) - passthrough: true - } - chRun = runIfBranch.run - chPassthrough = runIfBranch.passthrough - } else { - chRun = chModified - chPassthrough = Channel.empty() - } - - def chRunFiltered = workflowArgs.filter ? - chRun | filter{workflowArgs.filter(it)} : - chRun - - def chArgs = workflowArgs.fromState ? - chRunFiltered | map{ - def new_data = workflowArgs.fromState(it.take(2)) - [it[0], new_data] - } : - chRunFiltered | map {tup -> tup.take(2)} - - // fill in defaults - def chArgsWithDefaults = chArgs - | map { tuple -> - def id_ = tuple[0] - def data_ = tuple[1] - - // TODO: could move fromState to here - - // fetch default params from functionality - def defaultArgs = meta.config.allArguments - .findAll { it.containsKey("default") } - .collectEntries { [ it.plainName, it.default ] } - - // fetch overrides in params - def paramArgs = meta.config.allArguments - .findAll { par -> - def argKey = key_ + "__" + par.plainName - params.containsKey(argKey) - } - .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } - - // fetch overrides in data - def dataArgs = meta.config.allArguments - .findAll { data_.containsKey(it.plainName) } - .collectEntries { [ it.plainName, data_[it.plainName] ] } - - // combine params - def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs - - // remove arguments with explicit null values - combinedArgs - .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} - - combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) - - [id_, combinedArgs] + tuple.drop(2) - } - - // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. - def chInitialOutputMulti = chArgsWithDefaults - | _debug(workflowArgs, "processed") - // run workflow - | innerWorkflowFactory(workflowArgs) - def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] - assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" - // Add a channel ID to the events, which designates the channel the event was emitted from as a running number - // This number is used to sort the events later when the events are gathered from across the channels. - def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> - def newChannel = channel - | map {tuple -> - assert tuple instanceof List : - "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + - " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + - " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" - - def newEvent = [channelIndex] + tuple - return newEvent - } - return newChannel - } - // Put the events into 1 channel, cover case where there is only one channel is emitted - def chInitialOutput = chInitialOutputList.size() > 1 ? \ - chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ - chInitialOutputListWithIndexedEvents[0] - def chInitialOutputProcessed = chInitialOutput - | map { tuple -> - def channelId = tuple[0] - def id_ = tuple[1] - def output_ = tuple[2] - - // see if output map contains metadata - def meta_ = - output_ instanceof Map && output_.containsKey("_meta") ? - output_["_meta"] : - [:] - def join_id = meta_.join_id ?: id_ - - // remove metadata - output_ = output_.findAll{k, v -> k != "_meta"} - - // check value types - output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) - - [join_id, channelId, id_, output_] - } - // | view{"chInitialOutput: ${it.take(3)}"} - - // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] - def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) - // input tuple format: [join_id, channel_id, id, output, prev_state, ...] - // output tuple format: [join_id, channel_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(2).take(3)) - tup.take(3) + [new_state] + tup.drop(5) - } - if (workflowArgs.auto.publish == "state") { - def chPublishFiles = chPublishWithPreviousState - // input tuple format: [join_id, channel_id, id, new_state, ...] - // output tuple format: [join_id, channel_id, id, new_state] - | map{ tup -> - tup.take(4) - } - - safeJoin(chPublishFiles, chArgsWithDefaults, key_) - // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(2).take(3) - } - | publishFilesByConfig(key: key_, config: meta.config) - } - // Join the state from the events that were emitted from different channels - def chJoined = chInitialOutputProcessed - | map {tuple -> - def join_id = tuple[0] - def channel_id = tuple[1] - def id = tuple[2] - def other = tuple.drop(3) - // Below, groupTuple is used to join the events. To make sure resuming a workflow - // keeps working, the output state must be deterministic. This means the state needs to be - // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', - // but hashing the state when it is large can be problematic in terms of performance. - // Therefore, a custom comparator function is provided. We add the channel ID to the - // states so that we can use the channel ID to sort the items. - def stateWithChannelID = [[channel_id] * other.size(), other].transpose() - // A comparator that is provided to groupTuple's 'sort' argument is applied - // to all elements of the event tuple (that is not the 'id'). The comparator - // closure that is used below expects the input to be List. So the join_id and - // channel_id must also be wrapped in a list. - [[join_id], [channel_id], id] + stateWithChannelID - } - | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) - | map {join_ids, _, id, statesWithChannelID -> - // Remove the channel IDs from the states - def states = statesWithChannelID.collect{it[1]} - def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} - assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." - def newJoinIdUnique = newJoinId[0] - - // Merge the states from the different channels - def newState = states.inject([:]){ old_state, state_to_add -> - return old_state + state_to_add.collectEntries{k, v -> - if (!multipleArgs.contains(k)) { - // if the key is not a multiple argument, we expect only one value - if (old_state.containsKey(k)) { - assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." - } - [k, v] - } else { - // if the key is a multiple argument, append the different values into one list - def prevValue = old_state.getOrDefault(k, []) - def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] - [k, prevValueAsList + v] - } - } - } - - _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) - - // simplify output if need be - if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { - newState = newState.values()[0] - } - - return [newJoinIdUnique, id, newState] - } - - // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] - def chNewState = safeJoin(chJoined, chRunFiltered, key_) - // input tuple format: [join_id, id, output, prev_state, ...] - // output tuple format: [join_id, id, new_state, ...] - | map{ tup -> - def new_state = workflowArgs.toState(tup.drop(1).take(3)) - tup.take(2) + [new_state] + tup.drop(4) - } - - if (workflowArgs.auto.publish == "state") { - def chPublishStates = chNewState - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [join_id, id, new_state] - | map{ tup -> - tup.take(3) - } - - safeJoin(chPublishStates, chArgsWithDefaults, key_) - // input tuple format: [join_id, id, new_state, orig_state, ...] - // output tuple format: [id, new_state, orig_state] - | map { tup -> - tup.drop(1).take(3) - } - | publishStatesByConfig(key: key_, config: meta.config) - } - chReturn = chNewState - | map { tup -> - // input tuple format: [join_id, id, new_state, ...] - // output tuple format: [id, new_state, ...] - tup.drop(1) - } - | _debug(workflowArgs, "output") - | concat(chPassthrough) - - emit: chReturn - } - - def wf = workflowInstance.cloneWithName(key_) - - // add factory function - wf.metaClass.run = { runArgs -> - workflowFactory(runArgs, workflowArgs, meta) - } - // add config to module for later introspection - wf.metaClass.config = meta.config - - return wf -} - -nextflow.enable.dsl=2 - -// START COMPONENT-SPECIFIC CODE - -// create meta object -meta = [ - "resources_dir": moduleDir.toRealPath().normalize(), - "config": processConfig(readJsonBlob('''{ - "name" : "qc", - "namespace" : "workflows/qc", - "version" : "disable-scrublet_build", - "authors" : [ - { - "name" : "Dries Schaumont", - "roles" : [ - "author", - "maintainer" - ], - "info" : { - "role" : "Core Team Member", - "links" : { - "email" : "dries@data-intuitive.com", - "github" : "DriesSchaumont", - "orcid" : "0000-0002-4389-0440", - "linkedin" : "dries-schaumont" - }, - "organizations" : [ - { - "name" : "Data Intuitive", - "href" : "https://www.data-intuitive.com", - "role" : "Data Scientist" - } - ] - } - } - ], - "argument_groups" : [ - { - "name" : "Inputs", - "arguments" : [ - { - "type" : "string", - "name" : "--id", - "description" : "ID of the sample.", - "example" : [ - "foo" - ], - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "file", - "name" : "--input", - "alternatives" : [ - "-i" - ], - "description" : "Path to the sample.", - "example" : [ - "input.h5mu" - ], - "must_exist" : true, - "create_parent" : true, - "required" : true, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--modality", - "description" : "Which modality to process.", - "default" : [ - "rna" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--layer", - "description" : "Layer to calculate qc metrics for.", - "example" : [ - "raw_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Mitochondrial & Ribosomal Gene Detection", - "arguments" : [ - { - "type" : "string", - "name" : "--var_gene_names", - "description" : ".var column name to be used to detect mitochondrial/ribosomal genes instead of .var_names (default if not set).\nGene names matching with the regex value from --mitochondrial_gene_regex or --ribosomal_gene_regex will be \nidentified as mitochondrial or ribosomal genes, respectively.\n", - "example" : [ - "gene_symbol" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--var_name_mitochondrial_genes", - "description" : "In which .var slot to store a boolean array corresponding the mitochondrial genes.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--obs_name_mitochondrial_fraction", - "description" : ".Obs slot to store the fraction of reads found to be mitochondrial. Defaults to 'fraction_' suffixed by the value of --var_name_mitochondrial_genes\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--mitochondrial_gene_regex", - "description" : "Regex string that identifies mitochondrial genes from --var_gene_names.\nBy default will detect human and mouse mitochondrial genes from a gene symbol.\n", - "default" : [ - "^[mM][tT]-" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--var_name_ribosomal_genes", - "description" : "In which .var slot to store a boolean array corresponding the ribosomal genes.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--obs_name_ribosomal_fraction", - "description" : "When specified, write the fraction of counts originating from ribosomal genes \n(based on --ribosomal_gene_regex) to an .obs column with the specified name.\nRequires --var_name_ribosomal_genes.\n", - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--ribosomal_gene_regex", - "description" : "Regex string that identifies ribosomal genes from --var_gene_names.\nBy default will detect human and mouse ribosomal genes from a gene symbol.\n", - "default" : [ - "^[Mm]?[Rr][Pp][LlSs]" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "QC metrics calculation options", - "arguments" : [ - { - "type" : "string", - "name" : "--var_qc_metrics", - "description" : "Keys to select a boolean (containing only True or False) column from .var.\nFor each cell, calculate the proportion of total values for genes which are labeled 'True', \ncompared to the total sum of the values for all genes. Defaults to the value from\n--var_name_mitochondrial_genes.\n", - "example" : [ - "ercc,highly_variable" - ], - "required" : false, - "direction" : "input", - "multiple" : true, - "multiple_sep" : "," - }, - { - "type" : "integer", - "name" : "--top_n_vars", - "description" : "Number of top vars to be used to calculate cumulative proportions.\nIf not specified, proportions are not calculated. `--top_n_vars 20,50` finds\ncumulative proportion to the 20th and 50th most expressed vars.\n", - "default" : [ - 50, - 100, - 200, - 500 - ], - "required" : false, - "direction" : "input", - "multiple" : true, - "multiple_sep" : "," - }, - { - "type" : "string", - "name" : "--output_obs_num_nonzero_vars", - "description" : "Name of column in .obs describing, for each observation, the number of stored values\n(including explicit zeroes). In other words, the name of the column that counts\nfor each row the number of columns that contain data.\n", - "default" : [ - "num_nonzero_vars" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_obs_total_counts_vars", - "description" : "Name of the column for .obs describing, for each observation (row),\nthe sum of the stored values in the columns.\n", - "default" : [ - "total_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_num_nonzero_obs", - "description" : "Name of column describing, for each feature, the number of stored values\n(including explicit zeroes). In other words, the name of the column that counts\nfor each column the number of rows that contain data.\n", - "default" : [ - "num_nonzero_obs" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_total_counts_obs", - "description" : "Name of the column in .var describing, for each feature (column),\nthe sum of the stored values in the rows.\n", - "default" : [ - "total_counts" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_obs_mean", - "description" : "Name of the column in .obs providing the mean of the values in each row.\n", - "default" : [ - "obs_mean" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, - { - "type" : "string", - "name" : "--output_var_pct_dropout", - "description" : "Name of the column in .obs providing for each feature the percentage of\nobservations the feature does not appear on (i.e. is missing). Same as `--output_var_num_nonzero_obs`\nbut percentage based.\n", - "default" : [ - "pct_dropout" - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - } - ] - }, - { - "name" : "Outputs", - "arguments" : [ - { - "type" : "file", - "name" : "--output", - "description" : "Destination path to the output.", - "example" : [ - "output.h5mu" - ], - "must_exist" : true, - "create_parent" : true, - "required" : true, - "direction" : "output", - "multiple" : false, - "multiple_sep" : ";" - } - ] - } - ], - "resources" : [ - { - "type" : "nextflow_script", - "path" : "main.nf", - "is_executable" : true, - "entrypoint" : "run_wf" - }, - { - "type" : "file", - "path" : "/src/workflows/utils/" - }, - { - "type" : "file", - "path" : "/src/workflows/utils/labels.config", - "dest" : "nextflow_labels.config" - } - ], - "description" : "A pipeline to add basic qc statistics to a MuData ", - "test_resources" : [ - { - "type" : "nextflow_script", - "path" : "test.nf", - "is_executable" : true, - "entrypoint" : "test_wf" - }, - { - "type" : "file", - "path" : "/resources_test/concat_test_data" - }, - { - "type" : "file", - "path" : "/resources_test/pbmc_1k_protein_v3" - } - ], - "info" : { - "test_dependencies" : [ - { - "name" : "qc_test", - "namespace" : "test_workflows/qc" - } - ] - }, - "status" : "enabled", - "scope" : { - "image" : "public", - "target" : "public" - }, - "dependencies" : [ - { - "name" : "metadata/grep_annotation_column", - "repository" : { - "type" : "local" - } - }, - { - "name" : "qc/calculate_qc_metrics", - "repository" : { - "type" : "local" - } - } - ], - "license" : "MIT", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io" - }, - "runners" : [ - { - "type" : "nextflow", - "id" : "nextflow", - "directives" : { - "tag" : "$id" - }, - "auto" : { - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false - }, - "config" : { - "labels" : { - "mem1gb" : "memory = 1000000000.B", - "mem2gb" : "memory = 2000000000.B", - "mem5gb" : "memory = 5000000000.B", - "mem10gb" : "memory = 10000000000.B", - "mem20gb" : "memory = 20000000000.B", - "mem50gb" : "memory = 50000000000.B", - "mem100gb" : "memory = 100000000000.B", - "mem200gb" : "memory = 200000000000.B", - "mem500gb" : "memory = 500000000000.B", - "mem1tb" : "memory = 1000000000000.B", - "mem2tb" : "memory = 2000000000000.B", - "mem5tb" : "memory = 5000000000000.B", - "mem10tb" : "memory = 10000000000000.B", - "mem20tb" : "memory = 20000000000000.B", - "mem50tb" : "memory = 50000000000000.B", - "mem100tb" : "memory = 100000000000000.B", - "mem200tb" : "memory = 200000000000000.B", - "mem500tb" : "memory = 500000000000000.B", - "mem1gib" : "memory = 1073741824.B", - "mem2gib" : "memory = 2147483648.B", - "mem4gib" : "memory = 4294967296.B", - "mem8gib" : "memory = 8589934592.B", - "mem16gib" : "memory = 17179869184.B", - "mem32gib" : "memory = 34359738368.B", - "mem64gib" : "memory = 68719476736.B", - "mem128gib" : "memory = 137438953472.B", - "mem256gib" : "memory = 274877906944.B", - "mem512gib" : "memory = 549755813888.B", - "mem1tib" : "memory = 1099511627776.B", - "mem2tib" : "memory = 2199023255552.B", - "mem4tib" : "memory = 4398046511104.B", - "mem8tib" : "memory = 8796093022208.B", - "mem16tib" : "memory = 17592186044416.B", - "mem32tib" : "memory = 35184372088832.B", - "mem64tib" : "memory = 70368744177664.B", - "mem128tib" : "memory = 140737488355328.B", - "mem256tib" : "memory = 281474976710656.B", - "mem512tib" : "memory = 562949953421312.B", - "cpu1" : "cpus = 1", - "cpu2" : "cpus = 2", - "cpu5" : "cpus = 5", - "cpu10" : "cpus = 10", - "cpu20" : "cpus = 20", - "cpu50" : "cpus = 50", - "cpu100" : "cpus = 100", - "cpu200" : "cpus = 200", - "cpu500" : "cpus = 500", - "cpu1000" : "cpus = 1000" - }, - "script" : [ - "includeConfig(\\"nextflow_labels.config\\")" - ] - }, - "debug" : false, - "container" : "docker" - } - ], - "engines" : [ - { - "type" : "native", - "id" : "native" - } - ], - "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/qc/qc/config.vsh.yaml", - "runner" : "nextflow", - "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/qc/qc", - "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", - "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" - }, - "package_config" : { - "name" : "openpipeline", - "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", - "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", - "info" : { - "test_resources" : [ - { - "type" : "s3", - "path" : "s3://openpipelines-data", - "dest" : "resources_test" - } - ] - }, - "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", - "config_mods" : [ - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" - ], - "keywords" : [ - "single-cell", - "multimodal" - ], - "license" : "MIT", - "organization" : "openpipelines-bio", - "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline", - "docker_registry" : "ghcr.io", - "homepage" : "https://openpipelines.bio", - "documentation" : "https://openpipelines.bio/fundamentals", - "issue_tracker" : "https://github.com/openpipelines-bio/openpipeline/issues" - } - } -}''')) -] - -// resolve dependencies dependencies (if any) -meta["root_dir"] = getRootDir() -include { grep_annotation_column } from "${meta.resources_dir}/../../../../nextflow/metadata/grep_annotation_column/main.nf" -include { calculate_qc_metrics } from "${meta.resources_dir}/../../../../nextflow/qc/calculate_qc_metrics/main.nf" - -// inner workflow -// user-provided Nextflow code -workflow run_wf { - take: - input_ch - - main: - output_ch = input_ch - // Avoid conflict between output from component and output for this workflow - | map {id, state -> - assert state.output, "Output must be defined" - def new_state = state + ["workflow_output": state.output] - [id, new_state] - } - // Add default for var_qc_metrics component - | map {id, state -> - def var_qc_default = [] - // Remove the var_qc_metric argument from the state if its value is null (not specified) - def new_state = state.findAll { it.key != "var_qc_metrics" || it.value == null } - if (state.var_name_mitochondrial_genes) { - var_qc_default.add(state.var_name_mitochondrial_genes) - } - if (state.var_name_ribosomal_genes) { - var_qc_default.add(state.var_name_ribosomal_genes) - } - // Get the new state, but make sure to overwrite var_qc_metrics if the user has set it. - new_state = ["var_qc_metrics": var_qc_default.join(";")] + new_state - [id, new_state] - } - - | grep_annotation_column.run( - key: "grep_mitochondrial_genes", - runIf: { id, state -> - state.var_name_mitochondrial_genes - }, - fromState: { id, state -> - def stateMapping = [ - "input": state.input, - "modality": state.modality, - "input_column": state.var_gene_names, - "matrix": "var", - "output_match_column": state.var_name_mitochondrial_genes, - "regex_pattern": state.mitochondrial_gene_regex, - "input_layer": state.layer, - ] - stateMapping.output_fraction_column = state.obs_name_mitochondrial_fraction ? state.obs_name_mitochondrial_fraction: "fraction_$state.var_name_mitochondrial_genes" - return stateMapping - }, - toState: ["input": "output"] - ) - - | grep_annotation_column.run( - key: "grep_ribosomal_genes", - runIf: { id, state -> - state.var_name_ribosomal_genes - }, - fromState: { id, state -> - def stateMapping = [ - "input": state.input, - "modality": state.modality, - "input_column": state.var_gene_names, - "matrix": "var", - "output_match_column": state.var_name_ribosomal_genes, - "regex_pattern": state.ribosomal_gene_regex, - "input_layer": state.layer, - ] - stateMapping.output_fraction_column = state.obs_name_ribosomal_fraction ? state.obs_name_ribosomal_fraction: "fraction_$state.var_name_ribosomal_genes" - return stateMapping - }, - toState: ["input": "output"] - ) - - | calculate_qc_metrics.run( - fromState: { id, state -> - def newState = [ - "input": state.input, - "modality": state.modality, - "layer": state.layer, - // TODO: remove this workaround when Viash issue is resolved: - // 'top_n_vars': list(map(int, r''.split(';'))), - // ValueError: invalid literal for int() with base 10: '' - // See https://github.com/viash-io/viash/issues/619 - "top_n_vars": state.top_n_vars ? state.top_n_vars : null, - "var_qc_metrics_fill_na_value": state.var_qc_metrics_fill_na_value, - "output_obs_num_nonzero_vars": state.output_obs_num_nonzero_vars, - "output_obs_total_counts_vars": state.output_obs_total_counts_vars, - "output_var_num_nonzero_obs": state.output_var_num_nonzero_obs, - "output_var_total_counts_obs": state.output_var_total_counts_obs, - "output_var_obs_mean": state.output_var_obs_mean, - "output_var_pct_dropout": state.output_var_pct_dropout, - "output": state.workflow_output, - "compression": "gzip" - ] - if (state.var_qc_metrics) { - newState += ["var_qc_metrics": state.var_qc_metrics] - } - return newState - }, - toState: ["output": "output"] - ) - | setState(["output"]) - - emit: - output_ch -} - -// inner workflow hook -def innerWorkflowFactory(args) { - return run_wf -} - -// defaults -meta["defaults"] = [ - // key to be used to trace the process and determine output names - key: null, - - // fixed arguments to be passed to script - args: [:], - - // default directives - directives: readJsonBlob('''{ - "tag" : "$id" -}'''), - - // auto settings - auto: readJsonBlob('''{ - "simplifyInput" : true, - "simplifyOutput" : false, - "transcript" : false, - "publish" : false -}'''), - - // Apply a map over the incoming tuple - // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` - map: null, - - // Apply a map over the ID element of a tuple (i.e. the first element) - // Example: `{ id -> id + "_foo" }` - mapId: null, - - // Apply a map over the data element of a tuple (i.e. the second element) - // Example: `{ data -> [ input: data.output ] }` - mapData: null, - - // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) - // Example: `{ pt -> pt.drop(1) }` - mapPassthrough: null, - - // Filter the channel - // Example: `{ tup -> tup[0] == "foo" }` - filter: null, - - // Choose whether or not to run the component on the tuple if the condition is true. - // Otherwise, the tuple will be passed through. - // Example: `{ tup -> tup[0] != "skip_this" }` - runIf: null, - - // Rename keys in the data field of the tuple (i.e. the second element) - // Will likely be deprecated in favour of `fromState`. - // Example: `[ "new_key": "old_key" ]` - renameKeys: null, - - // Fetch data from the state and pass it to the module without altering the current state. - // - // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be passed to the module as is. - // - If it is a `List[String]`, the data will be the values of the state at the given keys. - // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. - // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. - // - // Example: `{ id, state -> [input: state.fastq_file] }` - // Default: `null` - fromState: null, - - // Determine how the state should be updated after the module has been run. - // - // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. - // - // - If it is `null`, the state will be replaced with the output of the module. - // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. - // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. - // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. - // - // Example: `{ id, output, state -> state + [counts: state.output] }` - // Default: `{ id, output, state -> output }` - toState: null, - - // Whether or not to print debug messages - // Default: `false` - debug: false -] - -// initialise default workflow -meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) - -// add workflow to environment -nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) - -// anonymous workflow for running this module as a standalone -workflow { - // add id argument if it's not already in the config - // TODO: deep copy - def newConfig = deepClone(meta.config) - def newParams = deepClone(params) - - def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} - if (!argsContainsId) { - def idArg = [ - 'name': '--id', - 'required': false, - 'type': 'string', - 'description': 'A unique id for every entry.', - 'multiple': false - ] - newConfig.arguments.add(0, idArg) - newConfig = processConfig(newConfig) - } - if (!newParams.containsKey("id")) { - newParams.id = "run" - } - - helpMessage(newConfig) - - channelFromParams(newParams, newConfig) - // make sure id is not in the state if id is not in the args - | map {id, state -> - if (!argsContainsId) { - [id, state.findAll{k, v -> k != "id"}] - } else { - [id, state] - } - } - | meta.workflow.run( - auto: [ publish: "state" ] - ) -} - -// END COMPONENT-SPECIFIC CODE diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow.config deleted file mode 100644 index 1e8b39e..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow.config +++ /dev/null @@ -1,126 +0,0 @@ -manifest { - name = 'workflows/qc/qc' - mainScript = 'main.nf' - nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' - description = 'A pipeline to add basic qc statistics to a MuData ' - author = 'Dries Schaumont' -} - -process.container = 'nextflow/bash:latest' - -// detect tempdir -tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' -).toAbsolutePath() - -profiles { - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - docker { - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - singularity { - singularity.enabled = true - singularity.autoMounts = true - docker.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - podman { - podman.enabled = true - docker.enabled = false - singularity.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } - shifter { - shifter.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - charliecloud.enabled = false - } - charliecloud { - charliecloud.enabled = true - docker.enabled = false - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - } -} - -process{ - withLabel: mem1gb { memory = 1000000000.B } - withLabel: mem2gb { memory = 2000000000.B } - withLabel: mem5gb { memory = 5000000000.B } - withLabel: mem10gb { memory = 10000000000.B } - withLabel: mem20gb { memory = 20000000000.B } - withLabel: mem50gb { memory = 50000000000.B } - withLabel: mem100gb { memory = 100000000000.B } - withLabel: mem200gb { memory = 200000000000.B } - withLabel: mem500gb { memory = 500000000000.B } - withLabel: mem1tb { memory = 1000000000000.B } - withLabel: mem2tb { memory = 2000000000000.B } - withLabel: mem5tb { memory = 5000000000000.B } - withLabel: mem10tb { memory = 10000000000000.B } - withLabel: mem20tb { memory = 20000000000000.B } - withLabel: mem50tb { memory = 50000000000000.B } - withLabel: mem100tb { memory = 100000000000000.B } - withLabel: mem200tb { memory = 200000000000000.B } - withLabel: mem500tb { memory = 500000000000000.B } - withLabel: mem1gib { memory = 1073741824.B } - withLabel: mem2gib { memory = 2147483648.B } - withLabel: mem4gib { memory = 4294967296.B } - withLabel: mem8gib { memory = 8589934592.B } - withLabel: mem16gib { memory = 17179869184.B } - withLabel: mem32gib { memory = 34359738368.B } - withLabel: mem64gib { memory = 68719476736.B } - withLabel: mem128gib { memory = 137438953472.B } - withLabel: mem256gib { memory = 274877906944.B } - withLabel: mem512gib { memory = 549755813888.B } - withLabel: mem1tib { memory = 1099511627776.B } - withLabel: mem2tib { memory = 2199023255552.B } - withLabel: mem4tib { memory = 4398046511104.B } - withLabel: mem8tib { memory = 8796093022208.B } - withLabel: mem16tib { memory = 17592186044416.B } - withLabel: mem32tib { memory = 35184372088832.B } - withLabel: mem64tib { memory = 70368744177664.B } - withLabel: mem128tib { memory = 140737488355328.B } - withLabel: mem256tib { memory = 281474976710656.B } - withLabel: mem512tib { memory = 562949953421312.B } - withLabel: cpu1 { cpus = 1 } - withLabel: cpu2 { cpus = 2 } - withLabel: cpu5 { cpus = 5 } - withLabel: cpu10 { cpus = 10 } - withLabel: cpu20 { cpus = 20 } - withLabel: cpu50 { cpus = 50 } - withLabel: cpu100 { cpus = 100 } - withLabel: cpu200 { cpus = 200 } - withLabel: cpu500 { cpus = 500 } - withLabel: cpu1000 { cpus = 1000 } -} - -includeConfig("nextflow_labels.config") diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_params.yaml deleted file mode 100644 index 435f0c8..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_params.yaml +++ /dev/null @@ -1,33 +0,0 @@ -# Inputs -id: # please fill in - example: "foo" -input: # please fill in - example: "input.h5mu" -modality: "rna" -# layer: "raw_counts" - -# Mitochondrial & Ribosomal Gene Detection -# var_gene_names: "gene_symbol" -# var_name_mitochondrial_genes: "foo" -# obs_name_mitochondrial_fraction: "foo" -mitochondrial_gene_regex: "^[mM][tT]-" -# var_name_ribosomal_genes: "foo" -# obs_name_ribosomal_fraction: "foo" -ribosomal_gene_regex: "^[Mm]?[Rr][Pp][LlSs]" - -# QC metrics calculation options -# var_qc_metrics: ["ercc,highly_variable"] -top_n_vars: [50, 100, 200, 500] -output_obs_num_nonzero_vars: "num_nonzero_vars" -output_obs_total_counts_vars: "total_counts" -output_var_num_nonzero_obs: "num_nonzero_obs" -output_var_total_counts_obs: "total_counts" -output_var_obs_mean: "obs_mean" -output_var_pct_dropout: "pct_dropout" - -# Outputs -# output: "$id.$key.output.h5mu" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_schema.json b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_schema.json deleted file mode 100644 index de611f2..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/nextflow_schema.json +++ /dev/null @@ -1,204 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": "qc", - "description": "A pipeline to add basic qc statistics to a MuData ", - "type": "object", - "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, - "inputs": { - "title": "Inputs", - "type": "object", - "description": "No description", - "properties": { - "id": { - "type": "string", - "description": "ID of the sample.", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"foo\"`. " - }, - "input": { - "type": "string", - "format": "path", - "exists": true, - "description": "Path to the sample.", - "help_text": "Type: `file`, multiple: `False`, required, direction: `input`, example: `\"input.h5mu\"`. " - }, - "modality": { - "type": "string", - "description": "Which modality to process.", - "help_text": "Type: `string`, multiple: `False`, default: `\"rna\"`. ", - "default": "rna" - }, - "layer": { - "type": "string", - "description": "Layer to calculate qc metrics for.", - "help_text": "Type: `string`, multiple: `False`, example: `\"raw_counts\"`. " - } - } - }, - "outputs": { - "title": "Outputs", - "type": "object", - "description": "No description", - "properties": { - "output": { - "type": "string", - "format": "path", - "description": "Destination path to the output.", - "help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output.h5mu\"`, direction: `output`, example: `\"output.h5mu\"`. ", - "default": "$id.$key.output.h5mu" - } - } - }, - "mitochondrial & ribosomal gene detection": { - "title": "Mitochondrial & Ribosomal Gene Detection", - "type": "object", - "description": "No description", - "properties": { - "var_gene_names": { - "type": "string", - "description": ".var column name to be used to detect mitochondrial/ribosomal genes instead of .var_names (default if not set).\nGene names matching with the regex value from --mitochondrial_gene_regex or --ribosomal_gene_regex will be \nidentified as mitochondrial or ribosomal genes, respectively.\n", - "help_text": "Type: `string`, multiple: `False`, example: `\"gene_symbol\"`. " - }, - "var_name_mitochondrial_genes": { - "type": "string", - "description": "In which .var slot to store a boolean array corresponding the mitochondrial genes.\n", - "help_text": "Type: `string`, multiple: `False`. " - }, - "obs_name_mitochondrial_fraction": { - "type": "string", - "description": ".Obs slot to store the fraction of reads found to be mitochondrial", - "help_text": "Type: `string`, multiple: `False`. " - }, - "mitochondrial_gene_regex": { - "type": "string", - "description": "Regex string that identifies mitochondrial genes from --var_gene_names.\nBy default will detect human and mouse mitochondrial genes from a gene symbol.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"^[mM][tT]-\"`. ", - "default": "^[mM][tT]-" - }, - "var_name_ribosomal_genes": { - "type": "string", - "description": "In which .var slot to store a boolean array corresponding the ribosomal genes.\n", - "help_text": "Type: `string`, multiple: `False`. " - }, - "obs_name_ribosomal_fraction": { - "type": "string", - "description": "When specified, write the fraction of counts originating from ribosomal genes \n(based on --ribosomal_gene_regex) to an .obs column with the specified name.\nRequires --var_name_ribosomal_genes.\n", - "help_text": "Type: `string`, multiple: `False`. " - }, - "ribosomal_gene_regex": { - "type": "string", - "description": "Regex string that identifies ribosomal genes from --var_gene_names.\nBy default will detect human and mouse ribosomal genes from a gene symbol.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"^[Mm]?[Rr][Pp][LlSs]\"`. ", - "default": "^[Mm]?[Rr][Pp][LlSs]" - } - } - }, - "qc metrics calculation options": { - "title": "QC metrics calculation options", - "type": "object", - "description": "No description", - "properties": { - "var_qc_metrics": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Keys to select a boolean (containing only True or False) column from .var.\nFor each cell, calculate the proportion of total values for genes which are labeled 'True', \ncompared to the total sum of the values for all genes", - "help_text": "Type: `string`, multiple: `True`, example: `[\"ercc,highly_variable\"]`. " - }, - "top_n_vars": { - "type": "array", - "items": { - "type": "integer" - }, - "description": "Number of top vars to be used to calculate cumulative proportions.\nIf not specified, proportions are not calculated", - "help_text": "Type: `integer`, multiple: `True`, default: `[50,100,200,500]`. ", - "default": [ - 50, - 100, - 200, - 500 - ] - }, - "output_obs_num_nonzero_vars": { - "type": "string", - "description": "Name of column in .obs describing, for each observation, the number of stored values\n(including explicit zeroes)", - "help_text": "Type: `string`, multiple: `False`, default: `\"num_nonzero_vars\"`. ", - "default": "num_nonzero_vars" - }, - "output_obs_total_counts_vars": { - "type": "string", - "description": "Name of the column for .obs describing, for each observation (row),\nthe sum of the stored values in the columns.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"total_counts\"`. ", - "default": "total_counts" - }, - "output_var_num_nonzero_obs": { - "type": "string", - "description": "Name of column describing, for each feature, the number of stored values\n(including explicit zeroes)", - "help_text": "Type: `string`, multiple: `False`, default: `\"num_nonzero_obs\"`. ", - "default": "num_nonzero_obs" - }, - "output_var_total_counts_obs": { - "type": "string", - "description": "Name of the column in .var describing, for each feature (column),\nthe sum of the stored values in the rows.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"total_counts\"`. ", - "default": "total_counts" - }, - "output_var_obs_mean": { - "type": "string", - "description": "Name of the column in .obs providing the mean of the values in each row.\n", - "help_text": "Type: `string`, multiple: `False`, default: `\"obs_mean\"`. ", - "default": "obs_mean" - }, - "output_var_pct_dropout": { - "type": "string", - "description": "Name of the column in .obs providing for each feature the percentage of\nobservations the feature does not appear on (i.e", - "help_text": "Type: `string`, multiple: `False`, default: `\"pct_dropout\"`. ", - "default": "pct_dropout" - } - } - }, - "nextflow input-output arguments": { - "title": "Nextflow input-output arguments", - "type": "object", - "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", - "properties": { - "publish_dir": { - "type": "string", - "description": "Path to an output directory.", - "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " - } - } - } - }, - "allOf": [ - { - "$ref": "#/$defs/inputs" - }, - { - "$ref": "#/$defs/outputs" - }, - { - "$ref": "#/$defs/mitochondrial & ribosomal gene detection" - }, - { - "$ref": "#/$defs/qc metrics calculation options" - }, - { - "$ref": "#/$defs/nextflow input-output arguments" - } - ] -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_params.yaml deleted file mode 100644 index 5ff4291..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_params.yaml +++ /dev/null @@ -1,36 +0,0 @@ -# Inputs -id: # please fill in - example: "concatenated" -input: # please fill in - example: "dataset.h5mu" -modality: "rna" -# layer: "foo" - -# Output -# output: "$id.$key.output.h5mu" - -# Filtering highly variable features -highly_variable_features_var_output: "filter_with_hvg" -highly_variable_features_obs_batch_key: "sample_id" -highly_variable_features_flavor: "seurat" -# highly_variable_features_n_top_features: 123 - -# QC metrics calculation options -var_qc_metrics: ["filter_with_hvg"] -top_n_vars: [50, 100, 200, 500] -output_obs_num_nonzero_vars: "num_nonzero_vars" -output_obs_total_counts_vars: "total_counts" -output_var_num_nonzero_obs: "num_nonzero_obs" -output_var_total_counts_obs: "total_counts" -output_var_obs_mean: "obs_mean" -output_var_pct_dropout: "pct_dropout" - -# RNA Scaling options -enable_scaling: false -scaling_output_layer: "scaled" -# scaling_max_value: 123.0 -scaling_zero_center: true - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_params.yaml b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_params.yaml deleted file mode 100644 index c5f6f64..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_params.yaml +++ /dev/null @@ -1,34 +0,0 @@ -# Input -id: # please fill in - example: "foo" -input: # please fill in - example: "dataset.h5mu" -# layer: "foo" - -# Output -# output: "$id.$key.output.h5mu" - -# Filtering options -# min_counts: 200 -# max_counts: 5000000 -# min_genes_per_cell: 200 -# max_genes_per_cell: 1500000 -# min_cells_per_gene: 3 -# min_fraction_mito: 0.0 -# max_fraction_mito: 0.2 -# min_fraction_ribo: 0.0 -# max_fraction_ribo: 0.2 -skip_scrublet_filtering: false - -# Mitochondrial & Ribosomal Gene Detection -# var_gene_names: "gene_symbol" -# var_name_mitochondrial_genes: "foo" -# obs_name_mitochondrial_fraction: "foo" -mitochondrial_gene_regex: "^[mM][tT]-" -# var_name_ribosomal_genes: "foo" -# obs_name_ribosomal_fraction: "foo" -ribosomal_gene_regex: "^[Mm]?[Rr][Pp][LlSs]" - -# Nextflow input-output arguments -publish_dir: # please fill in - example: "output/" -# param_list: "my_params.yaml" - -# Arguments diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config deleted file mode 100644 index 6b4b029..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config +++ /dev/null @@ -1 +0,0 @@ -process.errorStrategy = 'ignore' \ No newline at end of file diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config deleted file mode 100644 index 59d5b09..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config +++ /dev/null @@ -1,36 +0,0 @@ -profiles { - - // detect tempdir - tempDir = java.nio.file.Paths.get( - System.getenv('NXF_TEMP') ?: - System.getenv('VIASH_TEMP') ?: - System.getenv('TEMPDIR') ?: - System.getenv('TMPDIR') ?: - '/tmp' - ).toAbsolutePath() - - mount_temp { - docker.temp = tempDir - podman.temp = tempDir - charliecloud.temp = tempDir - } - - no_publish { - process { - withName: '.*' { - publishDir = [ - enabled: false - ] - } - } - } - - docker { - docker.enabled = true - // docker.userEmulation = true - singularity.enabled = false - podman.enabled = false - shifter.enabled = false - charliecloud.enabled = false - } -} \ No newline at end of file diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels.config deleted file mode 100644 index 8aca1a0..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels.config +++ /dev/null @@ -1,66 +0,0 @@ -process { - // Default resources for components that hardly do any processing - memory = { 2.GB * task.attempt } - cpus = 1 - - // Retry for exit codes that have something to do with memory issues - errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } - maxRetries = 3 - maxMemory = null - - // CPU resources - withLabel: singlecpu { cpus = 1 } - withLabel: lowcpu { cpus = 4 } - withLabel: midcpu { cpus = 10 } - withLabel: highcpu { cpus = 20 } - - // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } - - // Disk space - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } - - // NOTE: The above labels intentionally do not have an effect by default. - // The user should set the disk space requirements by adding the following - // to the compute environment: - // - // withLabel: lowdisk { disk = { 20.GB * task.attempt } } - // withLabel: middisk { disk = { 100.GB * task.attempt } } - // withLabel: highdisk { disk = { 200.GB * task.attempt } } - // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } -} - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config b/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config deleted file mode 100644 index aee866d..0000000 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config +++ /dev/null @@ -1,33 +0,0 @@ -process { - withLabel: lowmem { memory = 13.Gb } - withLabel: lowcpu { cpus = 4 } - withLabel: midmem { memory = 13.Gb } - withLabel: midcpu { cpus = 4 } - withLabel: highmem { memory = 13.Gb } - withLabel: highcpu { cpus = 4 } - withLabel: veryhighmem { memory = 13.Gb } - withLabel: lowdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: middisk { - disk = {process.disk ? process.disk : null} - } - withLabel: highdisk { - disk = {process.disk ? process.disk : null} - } - withLabel: veryhighdisk { - disk = {process.disk ? process.disk : null} - } -} - -env.NUMBA_CACHE_DIR = '/tmp' - -trace { - enabled = true - overwrite = true -} -dag { - overwrite = true -} - -process.maxForks = 1 diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml index 8e1d332..5115130 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "split_modalities" namespace: "workflows/multiomics" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -173,6 +173,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/multiomics/split_modalities/config.vsh.yaml" runner: "nextflow" @@ -180,13 +183,14 @@ build_info: output: "target/_private/nextflow/workflows/multiomics/split_modalities" executable: "target/_private/nextflow/workflows/multiomics/split_modalities/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/dataflow/split_modalities" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -203,6 +207,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -210,12 +218,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/main.nf index 56bca8a..5599cb5 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/main.nf @@ -1,4 +1,4 @@ -// split_modalities disable-scrublet_build +// split_modalities v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "split_modalities", "namespace" : "workflows/multiomics", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3269,17 +3269,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/multiomics/split_modalities/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/multiomics/split_modalities/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/_private/nextflow/workflows/multiomics/split_modalities", + "output" : "/workdir/root/repo/target/_private/nextflow/workflows/multiomics/split_modalities", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3289,21 +3290,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config index c7742c7..71f3be2 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/split_modalities' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'A pipeline to split a multimodal mudata files into several unimodal mudata files.' author = 'Dries Schaumont' } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/.config.vsh.yaml similarity index 93% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/.config.vsh.yaml index ed8c9e3..a7271fa 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "leiden" namespace: "cluster" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -214,7 +214,8 @@ engines: - type: "docker" id: "docker" image: "python:3.13-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -246,18 +247,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/cluster/leiden/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/cluster/leiden" executable: "target/nextflow/cluster/leiden/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -274,6 +278,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -281,13 +289,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/main.nf index 5682aaf..351e087 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/main.nf @@ -1,4 +1,4 @@ -// leiden disable-scrublet_build +// leiden v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "leiden", "namespace" : "cluster", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3293,7 +3293,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.13-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3338,20 +3339,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/cluster/leiden/config.vsh.yaml", + "config" : "/workdir/root/repo/src/cluster/leiden/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/cluster/leiden", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/cluster/leiden", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3361,22 +3367,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4128,9 +4141,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/cluster/leiden", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/cluster/leiden", + "tag" : "v3.0.0" }, "label" : [ "highcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow.config index ecfb7f8..ea05c9e 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'cluster/leiden' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Cluster cells using the [Leiden algorithm] [Traag18] implemented in the [Scanpy framework] [Wolf18]. \nLeiden is an improved version of the [Louvain algorithm] [Blondel08]. \nIt has been proposed for single-cell analysis by [Levine15] [Levine15]. \nThis requires having ran `neighbors/find_neighbors` or `neighbors/bbknn` first.\n\n[Blondel08]: Blondel et al. (2008), Fast unfolding of communities in large networks, J. Stat. Mech. \n[Levine15]: Levine et al. (2015), Data-Driven Phenotypic Dissection of AML Reveals Progenitor-like Cells that Correlate with Prognosis, Cell. \n[Traag18]: Traag et al. (2018), From Louvain to Leiden: guaranteeing well-connected communities arXiv. \n[Wolf18]: Wolf et al. (2018), Scanpy: large-scale single-cell gene expression data analysis, Genome Biology. \n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_schema.json similarity index 84% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_schema.json index b1b2c23..cb2d024 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Cluster cells using the [Leiden algorithm] [Traag18] implemented in the [Scanpy framework] [Wolf18]. \nLeiden is an improved version of the [Louvain algorithm] [Blondel08]. \nIt has been proposed for single-cell analysis by [Levine15] [Levine15]. \nThis requires having ran `neighbors/find_neighbors` or `neighbors/bbknn` first.\n\n[Blondel08]: Blondel et al. (2008), Fast unfolding of communities in large networks, J. Stat. Mech. \n[Levine15]: Levine et al. (2015), Data-Driven Phenotypic Dissection of AML Reveals Progenitor-like Cells that Correlate with Prognosis, Cell. \n[Traag18]: Traag et al. (2018), From Louvain to Leiden: guaranteeing well-connected communities arXiv. \n[Wolf18]: Wolf et al. (2018), Scanpy: large-scale single-cell gene expression data analysis, Genome Biology. \n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/cluster/leiden/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/cluster/leiden/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml index 3f6ec4f..943b94e 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "concatenate_h5mu" namespace: "dataflow" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -242,7 +242,8 @@ engines: - type: "docker" id: "docker" image: "python:3.11-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -278,18 +279,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/dataflow/concatenate_h5mu/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/dataflow/concatenate_h5mu" executable: "target/nextflow/dataflow/concatenate_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -306,6 +310,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -313,13 +321,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/main.nf index a40e83e..7be934b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/main.nf @@ -1,4 +1,4 @@ -// concatenate_h5mu disable-scrublet_build +// concatenate_h5mu v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "concatenate_h5mu", "namespace" : "dataflow", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3318,7 +3318,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.11-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3370,20 +3371,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/dataflow/concatenate_h5mu/config.vsh.yaml", + "config" : "/workdir/root/repo/src/dataflow/concatenate_h5mu/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/dataflow/concatenate_h5mu", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/dataflow/concatenate_h5mu", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3393,22 +3399,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4229,9 +4242,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/dataflow/concatenate_h5mu", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/dataflow/concatenate_h5mu", + "tag" : "v3.0.0" }, "label" : [ "midcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow.config index 911a5d7..418f5f0 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/concatenate_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Concatenate observations from samples in several (uni- and/or multi-modal) MuData files into a single file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json similarity index 87% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json index ddef9cf..d511fad 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Concatenate observations from samples in several (uni- and/or multi-modal) MuData files into a single file.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/concatenate_h5mu/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/concatenate_h5mu/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/.config.vsh.yaml similarity index 91% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/.config.vsh.yaml index 8734b10..724f826 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "merge" namespace: "dataflow" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -162,7 +162,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -192,18 +193,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/dataflow/merge/config.vsh.yml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/dataflow/merge" executable: "target/nextflow/dataflow/merge/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -220,6 +224,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -227,13 +235,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/main.nf index de9b3ac..37d2ca2 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/main.nf @@ -1,4 +1,4 @@ -// merge disable-scrublet_build +// merge v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "merge", "namespace" : "dataflow", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3245,7 +3245,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3288,20 +3289,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/dataflow/merge/config.vsh.yml", + "config" : "/workdir/root/repo/src/dataflow/merge/config.vsh.yml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/dataflow/merge", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/dataflow/merge", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3311,22 +3317,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3832,9 +3845,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/dataflow/merge", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/dataflow/merge", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow.config index 7f2b3de..892f36a 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/merge' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Combine one or more single-modality .h5mu files together into one .h5mu file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_schema.json similarity index 74% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_schema.json index 2b76e01..2f20aa0 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Combine one or more single-modality .h5mu files together into one .h5mu file.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/merge/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/merge/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/.config.vsh.yaml index a83b148..7a475e7 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "split_modalities" namespace: "dataflow" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -189,7 +189,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -219,18 +220,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/dataflow/split_modalities/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/dataflow/split_modalities" executable: "target/nextflow/dataflow/split_modalities/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -247,6 +251,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -254,13 +262,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/main.nf index 52bf93a..c72923a 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/main.nf @@ -1,4 +1,4 @@ -// split_modalities disable-scrublet_build +// split_modalities v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "split_modalities", "namespace" : "dataflow", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3279,7 +3279,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3322,20 +3323,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/dataflow/split_modalities/config.vsh.yaml", + "config" : "/workdir/root/repo/src/dataflow/split_modalities/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/dataflow/split_modalities", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/dataflow/split_modalities", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3345,22 +3351,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3850,9 +3863,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/dataflow/split_modalities", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/dataflow/split_modalities", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow.config index 2499612..1e03511 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/split_modalities' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Split the modalities from a single .h5mu multimodal sample into seperate .h5mu files. \n' author = 'Dries Schaumont, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_schema.json similarity index 77% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_schema.json index 20994cf..7e4a6d1 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Split the modalities from a single .h5mu multimodal sample into seperate .h5mu files. \n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dataflow/split_modalities/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dataflow/split_modalities/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/.config.vsh.yaml similarity index 93% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/.config.vsh.yaml index 39373ba..0dbcd09 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "pca" namespace: "dimred" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -239,7 +239,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -264,18 +265,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/dimred/pca/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/dimred/pca" executable: "target/nextflow/dimred/pca/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -292,6 +296,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -299,13 +307,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/main.nf index c4d3c9b..33b04b3 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/main.nf @@ -1,4 +1,4 @@ -// pca disable-scrublet_build +// pca v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "pca", "namespace" : "dimred", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3332,7 +3332,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3366,20 +3367,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/dimred/pca/config.vsh.yaml", + "config" : "/workdir/root/repo/src/dimred/pca/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/dimred/pca", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/dimred/pca", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3389,22 +3395,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3918,9 +3931,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/dimred/pca", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/dimred/pca", + "tag" : "v3.0.0" }, "label" : [ "highcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow.config index 9da224b..aaf7758 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dimred/pca' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Computes PCA coordinates, loadings and variance decomposition. Uses the implementation of scikit-learn [Pedregosa11].\n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_schema.json similarity index 85% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_schema.json index 6f2f45f..c9eb093 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Computes PCA coordinates, loadings and variance decomposition. Uses the implementation of scikit-learn [Pedregosa11].\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/pca/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/pca/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/.config.vsh.yaml index 2d65637..312bab8 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "umap" namespace: "dimred" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -293,7 +293,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -318,18 +319,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/dimred/umap/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/dimred/umap" executable: "target/nextflow/dimred/umap/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -346,6 +350,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -353,13 +361,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/main.nf index 91386ab..29e0ece 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/main.nf @@ -1,4 +1,4 @@ -// umap disable-scrublet_build +// umap v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "umap", "namespace" : "dimred", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3381,7 +3381,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3415,20 +3416,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/dimred/umap/config.vsh.yaml", + "config" : "/workdir/root/repo/src/dimred/umap/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/dimred/umap", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/dimred/umap", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3438,22 +3444,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3956,9 +3969,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/dimred/umap", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/dimred/umap", + "tag" : "v3.0.0" }, "label" : [ "highcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow.config index 30a14af..eaa7fb3 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dimred/umap' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'UMAP (Uniform Manifold Approximation and Projection) is a manifold learning technique suitable for visualizing high-dimensional data. Besides tending to be faster than tSNE, it optimizes the embedding such that it best reflects the topology of the data, which we represent throughout Scanpy using a neighborhood graph. tSNE, by contrast, optimizes the distribution of nearest-neighbor distances in the embedding such that these best match the distribution of distances in the high-dimensional space. We use the implementation of umap-learn [McInnes18]. For a few comparisons of UMAP with tSNE, see this preprint.\n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_schema.json similarity index 88% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_schema.json index f5ebf8e..d715953 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "UMAP (Uniform Manifold Approximation and Projection) is a manifold learning technique suitable for visualizing high-dimensional data. Besides tending to be faster than tSNE, it optimizes the embedding such that it best reflects the topology of the data, which we represent throughout Scanpy using a neighborhood graph. tSNE, by contrast, optimizes the distribution of nearest-neighbor distances in the embedding such that these best match the distribution of distances in the high-dimensional space. We use the implementation of umap-learn [McInnes18]. For a few comparisons of UMAP with tSNE, see this preprint.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/dimred/umap/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/dimred/umap/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml similarity index 95% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml index 7ee6207..92c9aae 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "highly_variable_features_scanpy" namespace: "feature_annotation" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -341,7 +341,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "python" @@ -369,18 +370,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/feature_annotation/highly_variable_features_scanpy/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/feature_annotation/highly_variable_features_scanpy" executable: "target/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -397,6 +401,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -404,13 +412,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf index e91d973..a83cf9f 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf @@ -1,4 +1,4 @@ -// highly_variable_features_scanpy disable-scrublet_build +// highly_variable_features_scanpy v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "highly_variable_features_scanpy", "namespace" : "feature_annotation", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3429,7 +3429,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3467,20 +3468,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/feature_annotation/highly_variable_features_scanpy/config.vsh.yaml", + "config" : "/workdir/root/repo/src/feature_annotation/highly_variable_features_scanpy/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/feature_annotation/highly_variable_features_scanpy", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/feature_annotation/highly_variable_features_scanpy", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3490,22 +3496,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4095,9 +4108,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/feature_annotation/highly_variable_features_scanpy", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/feature_annotation/highly_variable_features_scanpy", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config index 842ac2f..805aab7 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'feature_annotation/highly_variable_features_scanpy' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Annotate highly variable features [Satija15] [Zheng17] [Stuart19].\n\nExpects logarithmized data, except when flavor=\'seurat_v3\' in which count data is expected.\n\nDepending on flavor, this reproduces the R-implementations of Seurat [Satija15], Cell Ranger [Zheng17], and Seurat v3 [Stuart19].\n\nFor the dispersion-based methods ([Satija15] and [Zheng17]), the normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for features falling into a given bin for mean expression of features. This means that for each bin of mean expression, highly variable features are selected.\n\nFor [Stuart19], a normalized variance for each feature is computed. First, the data are standardized (i.e., z-score normalization per feature) with a regularized standard deviation. Next, the normalized variance is computed as the variance of each feature after the transformation. Features are ranked by the normalized variance.\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json similarity index 90% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json index 7e9f28e..83dab75 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Annotate highly variable features [Satija15] [Zheng17] [Stuart19].\n\nExpects logarithmized data, except when flavor='seurat_v3' in which count data is expected.\n\nDepending on flavor, this reproduces the R-implementations of Seurat [Satija15], Cell Ranger [Zheng17], and Seurat v3 [Stuart19].\n\nFor the dispersion-based methods ([Satija15] and [Zheng17]), the normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for features falling into a given bin for mean expression of features. This means that for each bin of mean expression, highly variable features are selected.\n\nFor [Stuart19], a normalized variance for each feature is computed. First, the data are standardized (i.e., z-score normalization per feature) with a regularized standard deviation. Next, the normalized variance is computed as the variance of each feature after the transformation. Features are ranked by the normalized variance.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/.config.vsh.yaml similarity index 93% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/.config.vsh.yaml index 661cc39..216395b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "delimit_fraction" namespace: "filter" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -218,7 +218,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -248,18 +249,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/filter/delimit_fraction/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/filter/delimit_fraction" executable: "target/nextflow/filter/delimit_fraction/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -276,6 +280,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -283,13 +291,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/main.nf index 013d096..af4c828 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/main.nf @@ -1,4 +1,4 @@ -// delimit_fraction disable-scrublet_build +// delimit_fraction v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "delimit_fraction", "namespace" : "filter", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3311,7 +3311,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3354,20 +3355,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/filter/delimit_fraction/config.vsh.yaml", + "config" : "/workdir/root/repo/src/filter/delimit_fraction/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/filter/delimit_fraction", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/filter/delimit_fraction", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3377,22 +3383,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3913,9 +3926,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/filter/delimit_fraction", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/filter/delimit_fraction", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow.config index fa3b040..c0a4ffe 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/delimit_fraction' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Turns a column containing values between 0 and 1 into a boolean column based on thresholds.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_schema.json similarity index 84% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_schema.json index ab932fb..52b00a1 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Turns a column containing values between 0 and 1 into a boolean column based on thresholds.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/delimit_fraction/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/delimit_fraction/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/.config.vsh.yaml index 59f8657..9b95f06 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "do_filter" namespace: "filter" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Robrecht Cannoodt" roles: @@ -192,7 +192,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -216,18 +217,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/filter/do_filter/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/filter/do_filter" executable: "target/nextflow/filter/do_filter/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -244,6 +248,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -251,13 +259,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/main.nf index 5938278..f536ada 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/main.nf @@ -1,4 +1,4 @@ -// do_filter disable-scrublet_build +// do_filter v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "do_filter", "namespace" : "filter", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Robrecht Cannoodt", @@ -3281,7 +3281,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3314,20 +3315,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/filter/do_filter/config.vsh.yaml", + "config" : "/workdir/root/repo/src/filter/do_filter/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/filter/do_filter", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/filter/do_filter", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3337,22 +3343,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3836,9 +3849,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/filter/do_filter", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/filter/do_filter", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow.config index a2c01c3..3f1e1a5 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/do_filter' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Remove observations and variables based on specified .obs and .var columns.\n' author = 'Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_schema.json similarity index 80% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_schema.json index b0c81cb..f63c9c3 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Remove observations and variables based on specified .obs and .var columns.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/do_filter/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/do_filter/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/.config.vsh.yaml index 1f31546..2669e7f 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "filter_with_counts" namespace: "filter" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -279,7 +279,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -303,18 +304,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/filter/filter_with_counts/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/filter/filter_with_counts" executable: "target/nextflow/filter/filter_with_counts/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -331,6 +335,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -338,13 +346,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/main.nf index ddfb2fa..fb7ae7f 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/main.nf @@ -1,4 +1,4 @@ -// filter_with_counts disable-scrublet_build +// filter_with_counts v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "filter_with_counts", "namespace" : "filter", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3391,7 +3391,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3424,20 +3425,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/filter/filter_with_counts/config.vsh.yaml", + "config" : "/workdir/root/repo/src/filter/filter_with_counts/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/filter/filter_with_counts", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/filter/filter_with_counts", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3447,22 +3453,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4012,9 +4025,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/filter/filter_with_counts", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/filter/filter_with_counts", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow.config index b5efa65..53cf9b9 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/filter_with_counts' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Filter scRNA-seq data based on the primary QC metrics. \nThis is based on both the UMI counts, the gene counts \nand the mitochondrial genes (genes starting with mt/MT).\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_schema.json similarity index 87% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_schema.json index ab3755c..4e586c5 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Filter scRNA-seq data based on the primary QC metrics. \nThis is based on both the UMI counts, the gene counts \nand the mitochondrial genes (genes starting with mt/MT).\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_counts/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_counts/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/.config.vsh.yaml similarity index 95% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/.config.vsh.yaml index bb515cc..d78e102 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "filter_with_scrublet" namespace: "filter" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -332,7 +332,8 @@ engines: - type: "docker" id: "docker" image: "python:3.10-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -366,18 +367,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/filter/filter_with_scrublet/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/filter/filter_with_scrublet" executable: "target/nextflow/filter/filter_with_scrublet/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -394,6 +398,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -401,13 +409,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/main.nf index 7222ce5..6ce2f64 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/main.nf @@ -1,4 +1,4 @@ -// filter_with_scrublet disable-scrublet_build +// filter_with_scrublet v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "filter_with_scrublet", "namespace" : "filter", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3428,7 +3428,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.10-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3475,20 +3476,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/filter/filter_with_scrublet/config.vsh.yaml", + "config" : "/workdir/root/repo/src/filter/filter_with_scrublet/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/filter/filter_with_scrublet", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/filter/filter_with_scrublet", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3498,22 +3504,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4050,9 +4063,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/filter/filter_with_scrublet", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/filter/filter_with_scrublet", + "tag" : "v3.0.0" }, "label" : [ "highcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow.config index 3a7f9d7..89e0b6b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/filter_with_scrublet' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Doublet detection using the Scrublet method (Wolock, Lopez and Klein, 2019).\nThe method tests for potential doublets by using the expression profiles of\ncells to generate synthetic potential doubles which are tested against cells. \nThe method returns a "doublet score" on which it calls for potential doublets.\n\nFor the source code please visit https://github.com/AllonKleinLab/scrublet.\n\nFor 10x we expect the doublet rates to be:\n Multiplet Rate (%) - # of Cells Loaded - # of Cells Recovered\n ~0.4% ~800 ~500\n ~0.8% ~1,600 ~1,000\n ~1.6% ~3,200 ~2,000\n ~2.3% ~4,800 ~3,000\n ~3.1% ~6,400 ~4,000\n ~3.9% ~8,000 ~5,000\n ~4.6% ~9,600 ~6,000\n ~5.4% ~11,200 ~7,000\n ~6.1% ~12,800 ~8,000\n ~6.9% ~14,400 ~9,000\n ~7.6% ~16,000 ~10,000\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_schema.json similarity index 90% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_schema.json index 2e99863..dae9f36 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Doublet detection using the Scrublet method (Wolock, Lopez and Klein, 2019).\nThe method tests for potential doublets by using the expression profiles of\ncells to generate synthetic potential doubles which are tested against cells. \nThe method returns a \"doublet score\" on which it calls for potential doublets.\n\nFor the source code please visit https://github.com/AllonKleinLab/scrublet.\n\nFor 10x we expect the doublet rates to be:\n Multiplet Rate (%) - # of Cells Loaded - # of Cells Recovered\n ~0.4% ~800 ~500\n ~0.8% ~1,600 ~1,000\n ~1.6% ~3,200 ~2,000\n ~2.3% ~4,800 ~3,000\n ~3.1% ~6,400 ~4,000\n ~3.9% ~8,000 ~5,000\n ~4.6% ~9,600 ~6,000\n ~5.4% ~11,200 ~7,000\n ~6.1% ~12,800 ~8,000\n ~6.9% ~14,400 ~9,000\n ~7.6% ~16,000 ~10,000\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/filter/filter_with_scrublet/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/filter/filter_with_scrublet/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/.config.vsh.yaml index 24eb365..da76554 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "add_id" namespace: "metadata" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -184,7 +184,8 @@ engines: - type: "docker" id: "docker" image: "python:3.11-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -214,18 +215,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/metadata/add_id/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/metadata/add_id" executable: "target/nextflow/metadata/add_id/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -242,6 +246,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -249,13 +257,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/main.nf index edb2a51..b73f113 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/main.nf @@ -1,4 +1,4 @@ -// add_id disable-scrublet_build +// add_id v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "add_id", "namespace" : "metadata", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3268,7 +3268,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.11-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3311,20 +3312,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/metadata/add_id/config.vsh.yaml", + "config" : "/workdir/root/repo/src/metadata/add_id/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/metadata/add_id", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/metadata/add_id", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3334,22 +3340,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3856,9 +3869,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/metadata/add_id", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/metadata/add_id", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow.config index fd157a2..3701e08 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'metadata/add_id' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Add id of .obs. Also allows to make .obs_names (the .obs index) unique \nby prefixing the values with an unique id per .h5mu file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_schema.json similarity index 79% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_schema.json index 695caee..83fcf94 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Add id of .obs. Also allows to make .obs_names (the .obs index) unique \nby prefixing the values with an unique id per .h5mu file.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/add_id/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/add_id/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml index 9c76997..3561b20 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "move_obsm_to_obs" namespace: "metadata" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -182,7 +182,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -212,18 +213,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/metadata/move_obsm_to_obs/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/metadata/move_obsm_to_obs" executable: "target/nextflow/metadata/move_obsm_to_obs/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -240,6 +244,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -247,13 +255,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/main.nf index 887dd6c..e060e28 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/main.nf @@ -1,4 +1,4 @@ -// move_obsm_to_obs disable-scrublet_build +// move_obsm_to_obs v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "move_obsm_to_obs", "namespace" : "metadata", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3267,7 +3267,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3310,20 +3311,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/metadata/move_obsm_to_obs/config.vsh.yaml", + "config" : "/workdir/root/repo/src/metadata/move_obsm_to_obs/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/metadata/move_obsm_to_obs", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/metadata/move_obsm_to_obs", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3333,22 +3339,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3845,9 +3858,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/metadata/move_obsm_to_obs", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/metadata/move_obsm_to_obs", + "tag" : "v3.0.0" }, "label" : [ "singlecpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow.config index b9e9557..5a0bf12 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'metadata/move_obsm_to_obs' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Move a matrix from .obsm to .obs. Newly created columns in .obs will \nbe created from the .obsm key suffixed with an underscore and the name of the columns\nof the specified .obsm matrix.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json similarity index 79% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json index 2bbbadc..44f1383 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Move a matrix from .obsm to .obs. Newly created columns in .obs will \nbe created from the .obsm key suffixed with an underscore and the name of the columns\nof the specified .obsm matrix.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "mudata input": { "title": "MuData Input", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/grep_annotation_column/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/metadata/move_obsm_to_obs/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/.config.vsh.yaml index fce0652..8a2acda 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "find_neighbors" namespace: "neighbors" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -295,7 +295,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -326,18 +327,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/neighbors/find_neighbors/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/neighbors/find_neighbors" executable: "target/nextflow/neighbors/find_neighbors/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -354,6 +358,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -361,13 +369,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/main.nf index 8559aab..4304dcc 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/main.nf @@ -1,4 +1,4 @@ -// find_neighbors disable-scrublet_build +// find_neighbors v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "find_neighbors", "namespace" : "neighbors", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3394,7 +3394,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3438,20 +3439,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/neighbors/find_neighbors/config.vsh.yaml", + "config" : "/workdir/root/repo/src/neighbors/find_neighbors/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/neighbors/find_neighbors", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/neighbors/find_neighbors", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3461,22 +3467,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3968,9 +3981,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/neighbors/find_neighbors", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/neighbors/find_neighbors", + "tag" : "v3.0.0" }, "label" : [ "lowcpu", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow.config index 2bcfb23..ffd620f 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'neighbors/find_neighbors' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Compute a neighborhood graph of observations [McInnes18].\n\nThe neighbor search efficiency of this heavily relies on UMAP [McInnes18], which also provides a method for estimating connectivities of data points - the connectivity of the manifold (method==\'umap\'). If method==\'gauss\', connectivities are computed according to [Coifman05], in the adaption of [Haghverdi16].\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_schema.json similarity index 88% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_schema.json index 2a7c568..1f81de2 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Compute a neighborhood graph of observations [McInnes18].\n\nThe neighbor search efficiency of this heavily relies on UMAP [McInnes18], which also provides a method for estimating connectivities of data points - the connectivity of the manifold (method=='umap'). If method=='gauss', connectivities are computed according to [Coifman05], in the adaption of [Haghverdi16].\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/metadata/move_obsm_to_obs/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/neighbors/find_neighbors/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/.config.vsh.yaml index 81d3542..12623a9 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "clr" namespace: "transform" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -201,7 +201,8 @@ engines: - type: "docker" id: "docker" image: "python:3.10-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -227,18 +228,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/transform/clr/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/transform/clr" executable: "target/nextflow/transform/clr/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -255,6 +259,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -262,13 +270,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/main.nf index bf654dc..f426bdf 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/main.nf @@ -1,4 +1,4 @@ -// clr disable-scrublet_build +// clr v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "clr", "namespace" : "transform", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3287,7 +3287,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.10-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3322,20 +3323,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/transform/clr/config.vsh.yaml", + "config" : "/workdir/root/repo/src/transform/clr/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/transform/clr", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/transform/clr", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3345,22 +3351,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3837,9 +3850,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/transform/clr", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/transform/clr", + "tag" : "v3.0.0" }, "label" : [ "lowmem", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow.config index 4fbfbca..e7d8be0 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/clr' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Perform CLR normalization on CITE-seq data (Stoeckius et al., 2017).\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_schema.json similarity index 80% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_schema.json index 294719c..cf13db6 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/clr/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Perform CLR normalization on CITE-seq data (Stoeckius et al., 2017).\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/.config.vsh.yaml index 505d2dd..a05efc2 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "delete_layer" namespace: "transform" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -185,7 +185,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -215,18 +216,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/transform/delete_layer/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/transform/delete_layer" executable: "target/nextflow/transform/delete_layer/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -243,6 +247,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -250,13 +258,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/main.nf index 5928cb5..ac3869c 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/main.nf @@ -1,4 +1,4 @@ -// delete_layer disable-scrublet_build +// delete_layer v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "delete_layer", "namespace" : "transform", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3272,7 +3272,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3315,20 +3316,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/transform/delete_layer/config.vsh.yaml", + "config" : "/workdir/root/repo/src/transform/delete_layer/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/transform/delete_layer", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/transform/delete_layer", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3338,22 +3344,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3829,9 +3842,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/transform/delete_layer", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/transform/delete_layer", + "tag" : "v3.0.0" }, "label" : [ "midmem", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow.config index 7cc4ec7..4b820cb 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/delete_layer' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Delete an anndata layer from one or more modalities.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_schema.json similarity index 79% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_schema.json index 3d305db..c2a74fc 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Delete an anndata layer from one or more modalities.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/neighbors/find_neighbors/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/delete_layer/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/.config.vsh.yaml index 2a12884..7b31838 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "log1p" namespace: "transform" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -215,7 +215,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -240,18 +241,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/transform/log1p/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/transform/log1p" executable: "target/nextflow/transform/log1p/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -268,6 +272,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -275,13 +283,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/clr/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/main.nf index 26445c0..f22350e 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/main.nf @@ -1,4 +1,4 @@ -// log1p disable-scrublet_build +// log1p v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "log1p", "namespace" : "transform", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3314,7 +3314,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3348,20 +3349,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/transform/log1p/config.vsh.yaml", + "config" : "/workdir/root/repo/src/transform/log1p/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/transform/log1p", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/transform/log1p", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3371,22 +3377,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3863,9 +3876,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/transform/log1p", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/transform/log1p", + "tag" : "v3.0.0" }, "label" : [ "midmem", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow.config index 41b3fac..da80aa7 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/log1p' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Logarithmize the data matrix. Computes X = log(X + 1), where log denotes the natural logarithm unless a different base is given.\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_schema.json similarity index 79% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_schema.json index 2823e2e..86612bf 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Logarithmize the data matrix. Computes X = log(X + 1), where log denotes the natural logarithm unless a different base is given.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/qc/calculate_qc_metrics/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/log1p/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/.config.vsh.yaml similarity index 93% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/.config.vsh.yaml index f9ae8d6..c2afe73 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "normalize_total" namespace: "transform" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -227,7 +227,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -258,18 +259,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/transform/normalize_total/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/transform/normalize_total" executable: "target/nextflow/transform/normalize_total/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -286,6 +290,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -293,13 +301,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/main.nf index 00d9567..4378e56 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/main.nf @@ -1,4 +1,4 @@ -// normalize_total disable-scrublet_build +// normalize_total v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "normalize_total", "namespace" : "transform", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3317,7 +3317,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3361,20 +3362,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/transform/normalize_total/config.vsh.yaml", + "config" : "/workdir/root/repo/src/transform/normalize_total/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/transform/normalize_total", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/transform/normalize_total", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3384,22 +3390,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3880,9 +3893,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/transform/normalize_total", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/transform/normalize_total", + "tag" : "v3.0.0" }, "label" : [ "midmem", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow.config index 3760ba2..4d597d6 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/normalize_total' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Normalize counts per cell.\n\nNormalize each cell by total counts over all genes, so that every cell has the same total count after normalization. If choosing target_sum=1e6, this is CPM normalization.\n\nIf exclude_highly_expressed=True, very highly expressed genes are excluded from the computation of the normalization factor (size factor) for each cell. This is meaningful as these can strongly influence the resulting normalized values for all other genes [Weinreb17].\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_schema.json similarity index 83% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_schema.json index 1d03fdb..2592614 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/normalize_total/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Normalize counts per cell.\n\nNormalize each cell by total counts over all genes, so that every cell has the same total count after normalization. If choosing target_sum=1e6, this is CPM normalization.\n\nIf exclude_highly_expressed=True, very highly expressed genes are excluded from the computation of the normalization factor (size factor) for each cell. This is meaningful as these can strongly influence the resulting normalized values for all other genes [Weinreb17].\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/delete_layer/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/normalize_total/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/.config.vsh.yaml similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/.config.vsh.yaml index bb7378f..7c2bf31 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "scale" namespace: "transform" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -203,7 +203,8 @@ engines: - type: "docker" id: "docker" image: "python:3.12-slim" - target_tag: "disable-scrublet_build" + target_registry: "images.viash-hub.com" + target_tag: "v3.0.0" namespace_separator: "/" setup: - type: "apt" @@ -234,18 +235,21 @@ engines: upgrade: true entrypoint: [] cmd: null +- type: "native" + id: "native" build_info: config: "src/transform/scale/config.vsh.yaml" runner: "nextflow" - engine: "docker" + engine: "docker|native" output: "target/nextflow/transform/scale" executable: "target/nextflow/transform/scale/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -262,6 +266,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -269,13 +277,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" - - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/compress_h5mu.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/compress_h5mu.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/main.nf index a8261ce..51b7157 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/main.nf @@ -1,4 +1,4 @@ -// scale disable-scrublet_build +// scale v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "scale", "namespace" : "transform", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3290,7 +3290,8 @@ meta = [ "type" : "docker", "id" : "docker", "image" : "python:3.12-slim", - "target_tag" : "disable-scrublet_build", + "target_registry" : "images.viash-hub.com", + "target_tag" : "v3.0.0", "namespace_separator" : "/", "setup" : [ { @@ -3334,20 +3335,25 @@ meta = [ "upgrade" : true } ] + }, + { + "type" : "native", + "id" : "native" } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/transform/scale/config.vsh.yaml", + "config" : "/workdir/root/repo/src/transform/scale/config.vsh.yaml", "runner" : "nextflow", - "engine" : "docker", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/transform/scale", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/transform/scale", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3357,22 +3363,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"", - ".engines[.type == 'docker'].target_tag := 'disable-scrublet_build'" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3858,9 +3871,9 @@ meta["defaults"] = [ // default directives directives: readJsonBlob('''{ "container" : { - "registry" : "ghcr.io", - "image" : "openpipelines-bio/openpipeline/transform/scale", - "tag" : "disable-scrublet_build" + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline/transform/scale", + "tag" : "v3.0.0" }, "label" : [ "lowmem", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow.config index f540337..6334300 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/scale' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Scale data to unit variance and zero mean.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_schema.json similarity index 80% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_schema.json index cf3c296..9d20750 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/scale/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Scale data to unit variance and zero mean.\n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "arguments": { "title": "Arguments", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/setup_logger.py similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/transform/log1p/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/transform/scale/setup_logger.py diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml similarity index 93% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml index 8782f02..18ab9d6 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "gdo_singlesample" namespace: "workflows/gdo" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -215,6 +215,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/gdo/gdo_singlesample/config.vsh.yaml" runner: "nextflow" @@ -222,14 +225,15 @@ build_info: output: "target/nextflow/workflows/gdo/gdo_singlesample" executable: "target/nextflow/workflows/gdo/gdo_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/filter/filter_with_counts" - "target/nextflow/filter/do_filter" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -246,6 +250,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -253,12 +261,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/main.nf index 65d33dc..ba1da68 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/main.nf @@ -1,4 +1,4 @@ -// gdo_singlesample disable-scrublet_build +// gdo_singlesample v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "gdo_singlesample", "namespace" : "workflows/gdo", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3322,17 +3322,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/gdo/gdo_singlesample/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/gdo/gdo_singlesample/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/gdo/gdo_singlesample", + "output" : "/workdir/root/repo/target/nextflow/workflows/gdo/gdo_singlesample", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3342,21 +3343,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow.config index 3c2d151..e429467 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/gdo/gdo_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Processing unimodal single-sample guide-derived oligonucleotide (GDO) data.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json similarity index 83% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json index f0d5250..1a0ffda 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Processing unimodal single-sample guide-derived oligonucleotide (GDO) data.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "input": { "title": "Input", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml index 576cf60..7debcce 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "dimensionality_reduction" namespace: "workflows/multiomics" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -268,6 +268,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/multiomics/dimensionality_reduction/config.vsh.yaml" runner: "nextflow" @@ -275,14 +278,15 @@ build_info: output: "target/nextflow/workflows/multiomics/dimensionality_reduction" executable: "target/nextflow/workflows/multiomics/dimensionality_reduction/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/dimred/pca" - "target/nextflow/workflows/multiomics/neighbors_leiden_umap" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -299,6 +303,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -306,12 +314,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/main.nf index ea15c5c..ac64ec9 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/main.nf @@ -1,4 +1,4 @@ -// dimensionality_reduction disable-scrublet_build +// dimensionality_reduction v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "dimensionality_reduction", "namespace" : "workflows/multiomics", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3385,17 +3385,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/multiomics/dimensionality_reduction/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/multiomics/dimensionality_reduction/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/multiomics/dimensionality_reduction", + "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/dimensionality_reduction", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3405,21 +3406,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config index 80da989..82425a1 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/dimensionality_reduction' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Run calculations that output information required for most integration methods: PCA, nearest neighbour and UMAP.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json similarity index 88% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json index 94eac3c..d241601 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Run calculations that output information required for most integration methods: PCA, nearest neighbour and UMAP.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml index 34a5339..0fa2065 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "neighbors_leiden_umap" namespace: "workflows/multiomics" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -224,6 +224,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/multiomics/neighbors_leiden_umap/config.vsh.yaml" runner: "nextflow" @@ -231,9 +234,9 @@ build_info: output: "target/nextflow/workflows/multiomics/neighbors_leiden_umap" executable: "target/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/cluster/leiden" - "target/nextflow/dimred/umap" @@ -241,6 +244,7 @@ build_info: - "target/nextflow/metadata/move_obsm_to_obs" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -257,6 +261,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -264,12 +272,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf index c34fb64..c604705 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf @@ -1,4 +1,4 @@ -// neighbors_leiden_umap disable-scrublet_build +// neighbors_leiden_umap v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "neighbors_leiden_umap", "namespace" : "workflows/multiomics", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3326,17 +3326,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/multiomics/neighbors_leiden_umap/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/multiomics/neighbors_leiden_umap/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/multiomics/neighbors_leiden_umap", + "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/neighbors_leiden_umap", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3346,21 +3347,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config index 649e6cb..c1f7c15 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/neighbors_leiden_umap' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Performs neighborhood search, leiden clustering and run umap on an integrated embedding.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json similarity index 86% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json index e70e9ca..5c8c869 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Performs neighborhood search, leiden clustering and run umap on an integrated embedding.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config similarity index 55% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config index 8aca1a0..94570ec 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config @@ -6,7 +6,10 @@ process { // Retry for exit codes that have something to do with memory issues errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } maxRetries = 3 - maxMemory = null + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] // CPU resources withLabel: singlecpu { cpus = 1 } @@ -15,10 +18,10 @@ process { withLabel: highcpu { cpus = 20 } // Memory resources - withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } } - withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } } - withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } - withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } // Disk space withLabel: lowdisk { @@ -43,24 +46,3 @@ process { // withLabel: highdisk { disk = { 200.GB * task.attempt } } // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } } - -def get_memory(to_compare) { - if (!process.containsKey("maxMemory") || !process.maxMemory) { - return to_compare - } - - try { - if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { - return process.maxMemory - } - else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { - return max_memory as nextflow.util.MemoryUnit - } - else { - return to_compare - } - } catch (all) { - println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" - System.exit(1) - } -} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml similarity index 96% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml index 26dee87..efd777b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "process_batches" namespace: "workflows/multiomics" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -370,6 +370,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/multiomics/process_batches/config.vsh.yaml" runner: "nextflow" @@ -377,9 +380,9 @@ build_info: output: "target/nextflow/workflows/multiomics/process_batches" executable: "target/nextflow/workflows/multiomics/process_batches/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/dataflow/merge" - "target/_private/nextflow/workflows/multiomics/split_modalities" @@ -390,6 +393,7 @@ build_info: - "target/nextflow/workflows/multiomics/dimensionality_reduction" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -406,6 +410,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -413,12 +421,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/main.nf index f9f8773..73f18ed 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/main.nf @@ -1,4 +1,4 @@ -// process_batches disable-scrublet_build +// process_batches v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "process_batches", "namespace" : "workflows/multiomics", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3508,17 +3508,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/multiomics/process_batches/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/multiomics/process_batches/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/multiomics/process_batches", + "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/process_batches", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3528,21 +3529,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow.config index b52c193..013fd8b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/process_batches' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'This workflow serves as an entrypoint into the \'full_pipeline\' in order to\nre-run the multisample processing and the integration setup. An input .h5mu file will \nfirst be split in order to run the multisample processing per modality. Next, the modalities\nare merged again and the integration setup pipeline is executed. Please note that this workflow\nassumes that samples from multiple pipelines are already concatenated. \n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_schema.json similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_schema.json index 9050614..c831375 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "This workflow serves as an entrypoint into the 'full_pipeline' in order to\nre-run the multisample processing and the integration setup. An input .h5mu file will \nfirst be split in order to run the multisample processing per modality. Next, the modalities\nare merged again and the integration setup pipeline is executed. Please note that this workflow\nassumes that samples from multiple pipelines are already concatenated. \n", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml similarity index 97% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml index 177b688..37cc7cc 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "process_samples" namespace: "workflows/multiomics" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -214,8 +214,8 @@ argument_groups: multiple: false multiple_sep: ";" - type: "boolean_true" - name: "--skip_scrublet_filtering" - description: "Skip the scrublet doublet removal step." + name: "--skip_scrublet_doublet_detection" + description: "Skip the scrublet doublet detection step." info: null direction: "input" - name: "CITE-seq filtering options" @@ -699,6 +699,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/multiomics/process_samples/config.vsh.yaml" runner: "nextflow" @@ -706,9 +709,9 @@ build_info: output: "target/nextflow/workflows/multiomics/process_samples" executable: "target/nextflow/workflows/multiomics/process_samples/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/metadata/add_id" - "target/_private/nextflow/workflows/multiomics/split_modalities" @@ -720,6 +723,7 @@ build_info: - "target/nextflow/workflows/multiomics/process_batches" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -736,6 +740,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -743,12 +751,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/main.nf index 09e5f48..ba70286 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/main.nf @@ -1,4 +1,4 @@ -// process_samples disable-scrublet_build +// process_samples v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "process_samples", "namespace" : "workflows/multiomics", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3297,8 +3297,8 @@ meta = [ }, { "type" : "boolean_true", - "name" : "--skip_scrublet_filtering", - "description" : "Skip the scrublet doublet removal step.", + "name" : "--skip_scrublet_doublet_detection", + "description" : "Skip the scrublet doublet detection step.", "direction" : "input" } ] @@ -3907,17 +3907,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/multiomics/process_samples/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/multiomics/process_samples/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/multiomics/process_samples", + "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/process_samples", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3927,21 +3928,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -4061,7 +4070,7 @@ workflow run_wf { "obs_name_ribosomal_fraction": "obs_name_ribosomal_fraction", "ribosomal_gene_regex": "ribosomal_gene_regex", "layer": "rna_layer", - "skip_scrublet_filtering": "skip_scrublet_filtering" + "skip_scrublet_doublet_detection": "skip_scrublet_doublet_detection" ], "prot": [ "min_counts": "prot_min_counts", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow.config index 1af1ef8..4842f7c 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/process_samples' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'A pipeline to analyse multiple multiomics samples.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_schema.json similarity index 95% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_schema.json index 689a9f9..dab191e 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "A pipeline to analyse multiple multiomics samples.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", @@ -141,9 +127,9 @@ "description": "Maximum fraction of UMIs that are mitochondrial.", "help_text": "Type: `double`, multiple: `False`, example: `0.2`. " }, - "skip_scrublet_filtering": { + "skip_scrublet_doublet_detection": { "type": "boolean", - "description": "Skip the scrublet doublet removal step.", + "description": "Skip the scrublet doublet detection step.", "help_text": "Type: `boolean_true`, multiple: `False`, default: `false`. ", "default": false } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml index 318dd87..5eee348 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "prot_multisample" namespace: "workflows/prot" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries Schaumont" roles: @@ -274,6 +274,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/prot/prot_multisample/config.vsh.yaml" runner: "nextflow" @@ -281,14 +284,15 @@ build_info: output: "target/nextflow/workflows/prot/prot_multisample" executable: "target/nextflow/workflows/prot/prot_multisample/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/transform/clr" - "target/nextflow/workflows/qc/qc" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -305,6 +309,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -312,12 +320,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/main.nf index 858e3c4..8ca52d9 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/main.nf @@ -1,4 +1,4 @@ -// prot_multisample disable-scrublet_build +// prot_multisample v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "prot_multisample", "namespace" : "workflows/prot", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries Schaumont", @@ -3379,17 +3379,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/prot/prot_multisample/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/prot/prot_multisample/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/prot/prot_multisample", + "output" : "/workdir/root/repo/target/nextflow/workflows/prot/prot_multisample", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3399,21 +3400,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow.config index 3deaa43..551701b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/prot/prot_multisample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Processing unimodal multi-sample ADT data.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_schema.json similarity index 89% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_schema.json index 54346ea..6402741 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Processing unimodal multi-sample ADT data.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml similarity index 94% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml index b18fe6b..006cc1b 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "prot_singlesample" namespace: "workflows/prot" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -245,6 +245,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/prot/prot_singlesample/config.vsh.yaml" runner: "nextflow" @@ -252,14 +255,15 @@ build_info: output: "target/nextflow/workflows/prot/prot_singlesample" executable: "target/nextflow/workflows/prot/prot_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/filter/filter_with_counts" - "target/nextflow/filter/do_filter" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -276,6 +280,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -283,12 +291,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/main.nf index 317a639..9fcde69 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/main.nf @@ -1,4 +1,4 @@ -// prot_singlesample disable-scrublet_build +// prot_singlesample v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "prot_singlesample", "namespace" : "workflows/prot", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3373,17 +3373,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/prot/prot_singlesample/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/prot/prot_singlesample/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/prot/prot_singlesample", + "output" : "/workdir/root/repo/target/nextflow/workflows/prot/prot_singlesample", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3393,21 +3394,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow.config index 165c6d2..a13d68a 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/prot/prot_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Processing unimodal single-sample CITE-seq data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json similarity index 82% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json index a182d89..988222c 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Processing unimodal single-sample CITE-seq data.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "input": { "title": "Input", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml similarity index 96% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml index ff7a547..50e0c4e 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "rna_multisample" namespace: "workflows/rna" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -418,6 +418,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/rna/rna_multisample/config.vsh.yaml" runner: "nextflow" @@ -425,9 +428,9 @@ build_info: output: "target/nextflow/workflows/rna/rna_multisample" executable: "target/nextflow/workflows/rna/rna_multisample/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/transform/normalize_total" - "target/nextflow/transform/log1p" @@ -438,6 +441,7 @@ build_info: - "target/nextflow/transform/scale" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -454,6 +458,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -461,12 +469,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/main.nf index d8d4d13..64a3604 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/main.nf @@ -1,4 +1,4 @@ -// rna_multisample disable-scrublet_build +// rna_multisample v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "rna_multisample", "namespace" : "workflows/rna", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3564,17 +3564,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/rna/rna_multisample/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/rna/rna_multisample/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/rna/rna_multisample", + "output" : "/workdir/root/repo/target/nextflow/workflows/rna/rna_multisample", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3584,21 +3585,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow.config index 45d59b7..ddc58d7 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/rna/rna_multisample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Processing unimodal multi-sample RNA transcriptomics data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_schema.json similarity index 92% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_schema.json index 49d6264..8d790f5 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Processing unimodal multi-sample RNA transcriptomics data.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "inputs": { "title": "Inputs", "type": "object", diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/qc/qc/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml similarity index 95% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml index 612fa94..9980e18 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "rna_singlesample" namespace: "workflows/rna" -version: "disable-scrublet_build" +version: "v3.0.0" authors: - name: "Dries De Maeyer" roles: @@ -195,8 +195,8 @@ argument_groups: multiple: false multiple_sep: ";" - type: "boolean_true" - name: "--skip_scrublet_filtering" - description: "Skip the scrublet doublet removal step." + name: "--skip_scrublet_doublet_detection" + description: "Skip the scrublet doublet detection step." info: null direction: "input" - name: "Mitochondrial & Ribosomal Gene Detection" @@ -386,6 +386,9 @@ runners: - "includeConfig(\"nextflow_labels.config\")" debug: false container: "docker" +engines: +- type: "native" + id: "native" build_info: config: "src/workflows/rna/rna_singlesample/config.vsh.yaml" runner: "nextflow" @@ -393,9 +396,9 @@ build_info: output: "target/nextflow/workflows/rna/rna_singlesample" executable: "target/nextflow/workflows/rna/rna_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "07297b53180b28c8e198414328683e941eec7ed0" + git_commit: "e92e56b49125af8ef2ebb11586191a6cbf9a8457" git_remote: "https://github.com/openpipelines-bio/openpipeline" - git_tag: "0.2.0-2044-g07297b53180" + git_tag: "0.2.0-2059-ge92e56b4" dependencies: - "target/nextflow/filter/filter_with_counts" - "target/nextflow/filter/filter_with_scrublet" @@ -404,6 +407,7 @@ build_info: - "target/nextflow/workflows/qc/qc" package_config: name: "openpipeline" + version: "v3.0.0" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -420,6 +424,10 @@ package_config: - type: "s3" path: "s3://openpipelines-data" dest: "resources_test" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -427,12 +435,14 @@ package_config: - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ )'" - - ".version := \"disable-scrublet_build\"" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'v3.0.0'" keywords: - "single-cell" - "multimodal" license: "MIT" - organization: "openpipelines-bio" + organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/main.nf similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/main.nf index 9dacd55..ced752f 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/main.nf @@ -1,4 +1,4 @@ -// rna_singlesample disable-scrublet_build +// rna_singlesample v3.0.0 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "rna_singlesample", "namespace" : "workflows/rna", - "version" : "disable-scrublet_build", + "version" : "v3.0.0", "authors" : [ { "name" : "Dries De Maeyer", @@ -3292,8 +3292,8 @@ meta = [ }, { "type" : "boolean_true", - "name" : "--skip_scrublet_filtering", - "description" : "Skip the scrublet doublet removal step.", + "name" : "--skip_scrublet_doublet_detection", + "description" : "Skip the scrublet doublet detection step.", "direction" : "input" } ] @@ -3536,17 +3536,18 @@ meta = [ } ], "build_info" : { - "config" : "/home/runner/work/openpipeline/openpipeline/src/workflows/rna/rna_singlesample/config.vsh.yaml", + "config" : "/workdir/root/repo/src/workflows/rna/rna_singlesample/config.vsh.yaml", "runner" : "nextflow", "engine" : "native", - "output" : "/home/runner/work/openpipeline/openpipeline/target/nextflow/workflows/rna/rna_singlesample", + "output" : "/workdir/root/repo/target/nextflow/workflows/rna/rna_singlesample", "viash_version" : "0.9.4", - "git_commit" : "07297b53180b28c8e198414328683e941eec7ed0", + "git_commit" : "e92e56b49125af8ef2ebb11586191a6cbf9a8457", "git_remote" : "https://github.com/openpipelines-bio/openpipeline", - "git_tag" : "0.2.0-2044-g07297b53180" + "git_tag" : "0.2.0-2059-ge92e56b4" }, "package_config" : { "name" : "openpipeline", + "version" : "v3.0.0", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3556,21 +3557,29 @@ meta = [ "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, "viash_version" : "0.9.4", - "source" : "/home/runner/work/openpipeline/openpipeline/src", - "target" : "/home/runner/work/openpipeline/openpipeline/target", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", "config_mods" : [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", - ".version := \\"disable-scrublet_build\\"" + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'v3.0.0'" ], "keywords" : [ "single-cell", "multimodal" ], "license" : "MIT", - "organization" : "openpipelines-bio", + "organization" : "vsh", "links" : { "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io", @@ -3744,7 +3753,9 @@ workflow run_wf { // from a modality. def stateMapping = [ input: state.input, - var_filter: ["filter_with_counts"] + var_filter: ["filter_with_counts"], + // If scrublet is skipped, the output should be set to the workflow output + output: state.workflow_output ] def obs_filter = ["filter_with_counts"] if (state.var_name_mitochondrial_genes) { @@ -3754,10 +3765,6 @@ workflow run_wf { obs_filter += ["filter_ribosomal"] } stateMapping += ["obs_filter": obs_filter] - // If scrublet is skipped, the output should be set to the workflow output - if (state.skip_scrublet_filtering) { - stateMapping += ["output": state.workflow_output] - } return stateMapping }, toState: ["input": "output"] @@ -3765,7 +3772,7 @@ workflow run_wf { // doublet calling | filter_with_scrublet.run( runIf: { id, state -> - !state.skip_scrublet_filtering + !state.skip_scrublet_doublet_detection }, fromState: [ "input": "input", @@ -3775,14 +3782,7 @@ workflow run_wf { args: [output_compression: "gzip"], toState: ["input": "output"] ) - // Make sure to use the correct ouput file names, - // irrespective of which component(s) (or combinations of then) - // were run. The above components - // should put their output into 'input' - | map {id, state -> - [id, ["workflow_output": state.input]] - } - | setState(["output": "workflow_output"]) + | setState(["output": "input"]) emit: output_ch diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow.config similarity index 99% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow.config index 3e7e25a..77574d9 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/rna/rna_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'disable-scrublet_build' + version = 'v3.0.0' description = 'Processing unimodal single-sample RNA transcriptomics data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json similarity index 89% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json index 96ee267..876b7f5 100644 --- a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json @@ -4,20 +4,6 @@ "description": "Processing unimodal single-sample RNA transcriptomics data.", "type": "object", "$defs": { - "Dataset input": { - "title": "Dataset input", - "type": "object", - "description": "Dataset input using nf-tower \"dataset\" or \"data explorer\". Allows for the input of multiple parameter sets to initialise a Nextflow channel.", - "properties": { - "param_list": { - "description": "Dataset input can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. The names of the input fields (e.g. csv columns, json keys) need to be an exact match with the workflow input parameters.", - "type": "string", - "default": "", - "format": "file-path", - "mimetype": "text/csv" - } - } - }, "input": { "title": "Input", "type": "object", @@ -106,9 +92,9 @@ "description": "Maximum fraction of UMIs that are ribosomal", "help_text": "Type: `double`, multiple: `False`, example: `0.2`. " }, - "skip_scrublet_filtering": { + "skip_scrublet_doublet_detection": { "type": "boolean", - "description": "Skip the scrublet doublet removal step.", + "description": "Skip the scrublet doublet detection step.", "help_text": "Type: `boolean_true`, multiple: `False`, default: `false`. ", "default": false } diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config diff --git a/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml index d9c5e4f..016c15f 100644 --- a/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cells2stats_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -256,7 +256,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -290,11 +290,11 @@ build_info: output: "target/executable/convert/from_cells2stats_to_h5mu" executable: "target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -314,7 +314,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu b/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu index 0cc4a25..defbf38 100755 --- a/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu +++ b/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_cells2stats_to_h5mu v0.1.0 +# from_cells2stats_to_h5mu v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -458,10 +458,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_cells2stats_to_h5mu" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:08Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -578,7 +578,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_cells2stats_to_h5mu v0.1.0" + echo "from_cells2stats_to_h5mu v0.1" echo "" echo "Convert spatial data resulting from Aviti Teton sequencers that have been" echo "processed by the Element Biosciences cells2stats workflow to H5MU format." @@ -722,7 +722,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_cells2stats_to_h5mu v0.1.0" + echo "from_cells2stats_to_h5mu v0.1" exit ;; --input) @@ -923,7 +923,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cells2stats_to_h5mu:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cells2stats_to_h5mu:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml index 0662ee9..ca00e7b 100644 --- a/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cosmx_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -184,7 +184,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -225,11 +225,11 @@ build_info: output: "target/executable/convert/from_cosmx_to_h5mu" executable: "target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -249,7 +249,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu b/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu index 66b46b9..2606ef7 100755 --- a/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu +++ b/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_cosmx_to_h5mu v0.1.0 +# from_cosmx_to_h5mu v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -459,10 +459,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_cosmx_to_h5mu" -LABEL org.opencontainers.image.created="2025-08-25T08:17:06Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:10Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -579,7 +579,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_cosmx_to_h5mu v0.1.0" + echo "from_cosmx_to_h5mu v0.1" echo "" echo "Converts the output from NanoString experiment into a MuData objcet." echo " - \`_exprMat_file.csv\`: File containing the counts." @@ -658,7 +658,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_cosmx_to_h5mu v0.1.0" + echo "from_cosmx_to_h5mu v0.1" exit ;; --input) @@ -805,7 +805,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cosmx_to_h5mu:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cosmx_to_h5mu:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml index 21d960c..f4948c1 100644 --- a/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cosmx_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -202,7 +202,7 @@ engines: id: "docker" image: "rocker/r2u:24.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -232,11 +232,11 @@ build_info: output: "target/executable/convert/from_cosmx_to_spatialexperiment" executable: "target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -256,7 +256,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment b/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment index a4c3da8..16d9a35 100755 --- a/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment +++ b/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_cosmx_to_spatialexperiment v0.1.0 +# from_cosmx_to_spatialexperiment v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -457,10 +457,10 @@ RUN Rscript -e 'options(warn = 2); if (!requireNamespace("BiocManager", quietly LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_cosmx_to_spatialexperiment" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -577,7 +577,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_cosmx_to_spatialexperiment v0.1.0" + echo "from_cosmx_to_spatialexperiment v0.1" echo "" echo "Creates a SpatialExperiment object from the downloaded unzipped CosMx directory" echo "for Nanostring" @@ -678,7 +678,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_cosmx_to_spatialexperiment v0.1.0" + echo "from_cosmx_to_spatialexperiment v0.1" exit ;; --input) @@ -853,7 +853,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cosmx_to_spatialexperiment:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_cosmx_to_spatialexperiment:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml index 3870daa..f391bf4 100644 --- a/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_h5mu_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -171,7 +171,7 @@ engines: id: "docker" image: "rocker/r2u:22.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -224,11 +224,11 @@ build_info: output: "target/executable/convert/from_h5mu_to_spatialexperiment" executable: "target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -248,7 +248,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment b/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment index bda5415..7ad3d08 100755 --- a/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment +++ b/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_h5mu_to_spatialexperiment v0.1.0 +# from_h5mu_to_spatialexperiment v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -458,10 +458,10 @@ RUN Rscript -e 'options(warn = 2); if (!requireNamespace("remotes", quietly = TR LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_h5mu_to_spatialexperiment" -LABEL org.opencontainers.image.created="2025-08-25T08:17:06Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -578,7 +578,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_h5mu_to_spatialexperiment v0.1.0" + echo "from_h5mu_to_spatialexperiment v0.1" echo "" echo "Converts an h5mu file into a SpatialExperiment object." echo "" @@ -650,7 +650,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_h5mu_to_spatialexperiment v0.1.0" + echo "from_h5mu_to_spatialexperiment v0.1" exit ;; --input) @@ -797,7 +797,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_h5mu_to_spatialexperiment:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_h5mu_to_spatialexperiment:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml index ec91206..15a713a 100644 --- a/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_spatialdata_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -180,7 +180,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -220,11 +220,11 @@ build_info: output: "target/executable/convert/from_spatialdata_to_h5mu" executable: "target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -244,7 +244,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu b/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu index 39e2227..f4def7a 100755 --- a/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu +++ b/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_spatialdata_to_h5mu v0.1.0 +# from_spatialdata_to_h5mu v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -459,10 +459,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_spatialdata_to_h5mu" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -579,7 +579,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_spatialdata_to_h5mu v0.1.0" + echo "from_spatialdata_to_h5mu v0.1" echo "" echo "Reads in the Tables field stored in a SpatialData object and converts it to an" echo "h5mu file." @@ -651,7 +651,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_spatialdata_to_h5mu v0.1.0" + echo "from_spatialdata_to_h5mu v0.1" exit ;; --input) @@ -798,7 +798,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_spatialdata_to_h5mu:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_spatialdata_to_h5mu:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml index 7516a91..2f3fc67 100644 --- a/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -198,7 +198,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -233,11 +233,11 @@ build_info: output: "target/executable/convert/from_xenium_to_h5mu" executable: "target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -257,7 +257,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu b/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu index 352055a..8cf6c65 100755 --- a/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu +++ b/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_xenium_to_h5mu v0.1.0 +# from_xenium_to_h5mu v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -458,10 +458,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_h5mu" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -578,7 +578,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_xenium_to_h5mu v0.1.0" + echo "from_xenium_to_h5mu v0.1" echo "" echo "Converts the output from Xenium to a single .h5mu file, where the count matrix" echo "is written to the \`rna\` modality." @@ -670,7 +670,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_xenium_to_h5mu v0.1.0" + echo "from_xenium_to_h5mu v0.1" exit ;; --input) @@ -839,7 +839,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_h5mu:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_h5mu:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml b/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml index 592da2e..4228e48 100644 --- a/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_spatialdata" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -278,7 +278,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -314,11 +314,11 @@ build_info: output: "target/executable/convert/from_xenium_to_spatialdata" executable: "target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -338,7 +338,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata b/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata index 969dc97..6c9609d 100755 --- a/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata +++ b/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_xenium_to_spatialdata v0.1.0 +# from_xenium_to_spatialdata v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -458,10 +458,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_spatialdata" -LABEL org.opencontainers.image.created="2025-08-25T08:17:06Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -578,7 +578,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_xenium_to_spatialdata v0.1.0" + echo "from_xenium_to_spatialdata v0.1" echo "" echo "Converts the output from 10X Genomics Xenium dataset into a SpatialData objcet." echo "By default, the following files will be converted:" @@ -711,7 +711,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_xenium_to_spatialdata v0.1.0" + echo "from_xenium_to_spatialdata v0.1" exit ;; --input) @@ -951,7 +951,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_spatialdata:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_spatialdata:v0.1' fi # print dockerfile diff --git a/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml index 5b3d7ba..6091805 100644 --- a/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -192,7 +192,7 @@ engines: id: "docker" image: "rocker/r2u:24.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -222,11 +222,11 @@ build_info: output: "target/executable/convert/from_xenium_to_spatialexperiment" executable: "target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -246,7 +246,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment b/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment index 55ffc78..daf5ea3 100755 --- a/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment +++ b/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# from_xenium_to_spatialexperiment v0.1.0 +# from_xenium_to_spatialexperiment v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -457,10 +457,10 @@ RUN Rscript -e 'options(warn = 2); if (!requireNamespace("BiocManager", quietly LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_spatialexperiment" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:09Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -577,7 +577,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "from_xenium_to_spatialexperiment v0.1.0" + echo "from_xenium_to_spatialexperiment v0.1" echo "" echo "Creates a SpatialExperiment object from the downloaded unzipped Xenium Output" echo "Bundle directory" @@ -673,7 +673,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "from_xenium_to_spatialexperiment v0.1.0" + echo "from_xenium_to_spatialexperiment v0.1" exit ;; --input) @@ -837,7 +837,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_spatialexperiment:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_xenium_to_spatialexperiment:v0.1' fi # print dockerfile diff --git a/target/executable/filter/subset_cosmx/.config.vsh.yaml b/target/executable/filter/subset_cosmx/.config.vsh.yaml index dcdc94b..d3ed1c4 100644 --- a/target/executable/filter/subset_cosmx/.config.vsh.yaml +++ b/target/executable/filter/subset_cosmx/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "subset_cosmx" namespace: "filter" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -191,7 +191,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -227,11 +227,11 @@ build_info: output: "target/executable/filter/subset_cosmx" executable: "target/executable/filter/subset_cosmx/subset_cosmx" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -251,7 +251,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/filter/subset_cosmx/subset_cosmx b/target/executable/filter/subset_cosmx/subset_cosmx index 8a12b3b..ba50f94 100755 --- a/target/executable/filter/subset_cosmx/subset_cosmx +++ b/target/executable/filter/subset_cosmx/subset_cosmx @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# subset_cosmx v0.1.0 +# subset_cosmx v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -458,10 +458,10 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component filter subset_cosmx" -LABEL org.opencontainers.image.created="2025-08-25T08:17:05Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:08Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -578,7 +578,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "subset_cosmx v0.1.0" + echo "subset_cosmx v0.1" echo "" echo "Filters the output from NanoString experiment to keep only a subset of the" echo "fields of view." @@ -666,7 +666,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "subset_cosmx v0.1.0" + echo "subset_cosmx v0.1" exit ;; --input) @@ -824,7 +824,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/filter/subset_cosmx:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/filter/subset_cosmx:v0.1' fi # print dockerfile diff --git a/target/executable/mapping/spaceranger_count/.config.vsh.yaml b/target/executable/mapping/spaceranger_count/.config.vsh.yaml index c33663a..53e9304 100644 --- a/target/executable/mapping/spaceranger_count/.config.vsh.yaml +++ b/target/executable/mapping/spaceranger_count/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "spaceranger_count" namespace: "mapping" -version: "v0.1.0" +version: "v0.1" authors: - name: "Jakub Majercik" roles: @@ -408,7 +408,7 @@ engines: id: "docker" image: "ghcr.io/data-intuitive/spaceranger:3.1" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "docker" @@ -426,11 +426,11 @@ build_info: output: "target/executable/mapping/spaceranger_count" executable: "target/executable/mapping/spaceranger_count/spaceranger_count" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -450,7 +450,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/executable/mapping/spaceranger_count/spaceranger_count b/target/executable/mapping/spaceranger_count/spaceranger_count index 12b0d0d..0a13943 100755 --- a/target/executable/mapping/spaceranger_count/spaceranger_count +++ b/target/executable/mapping/spaceranger_count/spaceranger_count @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# spaceranger_count v0.1.0 +# spaceranger_count v0.1 # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -453,10 +453,10 @@ apt upgrade -y && apt install -y procps && rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.authors="Jakub Majercik" LABEL org.opencontainers.image.description="Companion container for running component mapping spaceranger_count" -LABEL org.opencontainers.image.created="2025-08-25T08:17:06Z" +LABEL org.opencontainers.image.created="2025-08-25T11:47:10Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="963107088819638e9e15d4f413c0a763dc81a192" -LABEL org.opencontainers.image.version="v0.1.0" +LABEL org.opencontainers.image.revision="420b610b045c138f07475a1af39529f233d9af5a" +LABEL org.opencontainers.image.version="v0.1" VIASHDOCKER fi @@ -573,7 +573,7 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "spaceranger_count v0.1.0" + echo "spaceranger_count v0.1" echo "" echo "Count gene expression and protein expression reads from a single capture area." echo "" @@ -775,7 +775,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "spaceranger_count v0.1.0" + echo "spaceranger_count v0.1" exit ;; --gex_reference) @@ -1135,7 +1135,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/mapping/spaceranger_count:v0.1.0' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/mapping/spaceranger_count:v0.1' fi # print dockerfile diff --git a/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml index 569573b..abe0e83 100644 --- a/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cells2stats_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -256,7 +256,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -290,11 +290,11 @@ build_info: output: "target/nextflow/convert/from_cells2stats_to_h5mu" executable: "target/nextflow/convert/from_cells2stats_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -314,7 +314,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf b/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf index d79b12e..76eb16d 100644 --- a/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf +++ b/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf @@ -1,4 +1,4 @@ -// from_cells2stats_to_h5mu v0.1.0 +// from_cells2stats_to_h5mu v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_cells2stats_to_h5mu", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3336,7 +3336,7 @@ meta = [ "id" : "docker", "image" : "python:3.13-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3382,12 +3382,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cells2stats_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3412,7 +3412,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -4123,7 +4123,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_cells2stats_to_h5mu", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_cells2stats_to_h5mu/nextflow.config b/target/nextflow/convert/from_cells2stats_to_h5mu/nextflow.config index 98b884f..7a21211 100644 --- a/target/nextflow/convert/from_cells2stats_to_h5mu/nextflow.config +++ b/target/nextflow/convert/from_cells2stats_to_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_cells2stats_to_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Convert spatial data resulting from Aviti Teton sequencers that have been processed by the Element Biosciences cells2stats workflow to H5MU format.\n\nThis component processes cells2stats count matrices to create a standardized H5MU file for downstream analysis.\n\nThe component reads:\n- Parquet file containing the count matrix and metadata\n- Panel.json with target and batch information\n\nAnd outputs an H5MU file with:\n- Count data as the main .X matrix\n- Spatial coordinates in obsm\n- Cell Paint intensities in obsm (optional)\n- Nuclear count data as a layer (optional)\n- CellProfiler morphology metrics in obsm (optional)\n- Unassigned targets in obsm (optional)\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml index c0ad2a4..134d44d 100644 --- a/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cosmx_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -184,7 +184,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -225,11 +225,11 @@ build_info: output: "target/nextflow/convert/from_cosmx_to_h5mu" executable: "target/nextflow/convert/from_cosmx_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -249,7 +249,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_cosmx_to_h5mu/main.nf b/target/nextflow/convert/from_cosmx_to_h5mu/main.nf index 25a0a49..dfdffd3 100644 --- a/target/nextflow/convert/from_cosmx_to_h5mu/main.nf +++ b/target/nextflow/convert/from_cosmx_to_h5mu/main.nf @@ -1,4 +1,4 @@ -// from_cosmx_to_h5mu v0.1.0 +// from_cosmx_to_h5mu v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_cosmx_to_h5mu", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3274,7 +3274,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3331,12 +3331,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cosmx_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3361,7 +3361,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3830,7 +3830,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_cosmx_to_h5mu", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_cosmx_to_h5mu/nextflow.config b/target/nextflow/convert/from_cosmx_to_h5mu/nextflow.config index 6148394..ea494ab 100644 --- a/target/nextflow/convert/from_cosmx_to_h5mu/nextflow.config +++ b/target/nextflow/convert/from_cosmx_to_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_cosmx_to_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Converts the output from NanoString experiment into a MuData objcet.\n - `_exprMat_file.csv`: File containing the counts.\n - ``_metadata_file: File containing the spatial coordinates and additional cell-level metadata.\n - `_fov_file.csv`: File containing the coordinates of all the fields of view.\nIn addition to reading the regular Nanostring output, it loads CellComposite and CellLabels directories, if present,\ncontaining the images.\n' author = 'Dorien Roosen, Weiwei Schultz' } diff --git a/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml index fbdf920..1444962 100644 --- a/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_cosmx_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -202,7 +202,7 @@ engines: id: "docker" image: "rocker/r2u:24.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -232,11 +232,11 @@ build_info: output: "target/nextflow/convert/from_cosmx_to_spatialexperiment" executable: "target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -256,7 +256,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf b/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf index 3d838be..24d3fdd 100644 --- a/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf @@ -1,4 +1,4 @@ -// from_cosmx_to_spatialexperiment v0.1.0 +// from_cosmx_to_spatialexperiment v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_cosmx_to_spatialexperiment", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3280,7 +3280,7 @@ meta = [ "id" : "docker", "image" : "rocker/r2u:24.04", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3322,12 +3322,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cosmx_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3352,7 +3352,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3821,7 +3821,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_cosmx_to_spatialexperiment", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_cosmx_to_spatialexperiment/nextflow.config b/target/nextflow/convert/from_cosmx_to_spatialexperiment/nextflow.config index fd28d4a..e8267ca 100644 --- a/target/nextflow/convert/from_cosmx_to_spatialexperiment/nextflow.config +++ b/target/nextflow/convert/from_cosmx_to_spatialexperiment/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_cosmx_to_spatialexperiment' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Creates a SpatialExperiment object from the downloaded unzipped CosMx directory for Nanostring\nCosMx spatial gene expression data, and saves it as a SpatialExperiment object.\nThe constructor assumes the downloaded unzipped CosMx Folder has the following structure:\n\nMandatory files\n· | — *_exprMat_file.csv\n· | — *_metadata_file.csv\nOptional files, by default added to the metadata() as a list of paths (will be converted to parquet):\n· | —*_fov_positions_file.csv\n· | — *_tx_file.csv\n· | — *_polygons.csv\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml index 41adc92..274f497 100644 --- a/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_h5mu_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -171,7 +171,7 @@ engines: id: "docker" image: "rocker/r2u:22.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -224,11 +224,11 @@ build_info: output: "target/nextflow/convert/from_h5mu_to_spatialexperiment" executable: "target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -248,7 +248,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf b/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf index 96f55f9..069c2da 100644 --- a/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf @@ -1,4 +1,4 @@ -// from_h5mu_to_spatialexperiment v0.1.0 +// from_h5mu_to_spatialexperiment v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_h5mu_to_spatialexperiment", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3257,7 +3257,7 @@ meta = [ "id" : "docker", "image" : "rocker/r2u:22.04", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3332,12 +3332,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_h5mu_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3362,7 +3362,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3910,7 +3910,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_h5mu_to_spatialexperiment", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_h5mu_to_spatialexperiment/nextflow.config b/target/nextflow/convert/from_h5mu_to_spatialexperiment/nextflow.config index d87780a..627aa3a 100644 --- a/target/nextflow/convert/from_h5mu_to_spatialexperiment/nextflow.config +++ b/target/nextflow/convert/from_h5mu_to_spatialexperiment/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_h5mu_to_spatialexperiment' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Converts an h5mu file into a SpatialExperiment object.\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml index 413233e..a6fafae 100644 --- a/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_spatialdata_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -180,7 +180,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -220,11 +220,11 @@ build_info: output: "target/nextflow/convert/from_spatialdata_to_h5mu" executable: "target/nextflow/convert/from_spatialdata_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -244,7 +244,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf b/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf index 99dd1c1..3c17bf3 100644 --- a/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf +++ b/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf @@ -1,4 +1,4 @@ -// from_spatialdata_to_h5mu v0.1.0 +// from_spatialdata_to_h5mu v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_spatialdata_to_h5mu", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3274,7 +3274,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3330,12 +3330,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_spatialdata_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3360,7 +3360,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3814,7 +3814,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_spatialdata_to_h5mu", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_spatialdata_to_h5mu/nextflow.config b/target/nextflow/convert/from_spatialdata_to_h5mu/nextflow.config index c723588..03d48db 100644 --- a/target/nextflow/convert/from_spatialdata_to_h5mu/nextflow.config +++ b/target/nextflow/convert/from_spatialdata_to_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_spatialdata_to_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Reads in the Tables field stored in a SpatialData object and converts it to an h5mu file.\n \n' author = 'Dorien Roosen, Weiwei Schultz' } diff --git a/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml index 2b8761b..b316c65 100644 --- a/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_h5mu" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -198,7 +198,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -233,11 +233,11 @@ build_info: output: "target/nextflow/convert/from_xenium_to_h5mu" executable: "target/nextflow/convert/from_xenium_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -257,7 +257,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_xenium_to_h5mu/main.nf b/target/nextflow/convert/from_xenium_to_h5mu/main.nf index e42d64b..01ddd77 100644 --- a/target/nextflow/convert/from_xenium_to_h5mu/main.nf +++ b/target/nextflow/convert/from_xenium_to_h5mu/main.nf @@ -1,4 +1,4 @@ -// from_xenium_to_h5mu v0.1.0 +// from_xenium_to_h5mu v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_xenium_to_h5mu", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3284,7 +3284,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3331,12 +3331,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3361,7 +3361,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3862,7 +3862,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_xenium_to_h5mu", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_xenium_to_h5mu/nextflow.config b/target/nextflow/convert/from_xenium_to_h5mu/nextflow.config index 83579b7..ebc7fb8 100644 --- a/target/nextflow/convert/from_xenium_to_h5mu/nextflow.config +++ b/target/nextflow/convert/from_xenium_to_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_xenium_to_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Converts the output from Xenium to a single .h5mu file, where the count matrix is written to the `rna` modality.\nThe following files are expected to be present in the Xenium output bundle:\n├── cell_feature_matrix.h5\n├── cells.parquet\n├── experiment.xenium\n└── metrics_summary.csv\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml index 1e0a984..f8788a3 100644 --- a/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_spatialdata" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -278,7 +278,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -314,11 +314,11 @@ build_info: output: "target/nextflow/convert/from_xenium_to_spatialdata" executable: "target/nextflow/convert/from_xenium_to_spatialdata/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -338,7 +338,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_xenium_to_spatialdata/main.nf b/target/nextflow/convert/from_xenium_to_spatialdata/main.nf index 5e8fc6d..76d621e 100644 --- a/target/nextflow/convert/from_xenium_to_spatialdata/main.nf +++ b/target/nextflow/convert/from_xenium_to_spatialdata/main.nf @@ -1,4 +1,4 @@ -// from_xenium_to_spatialdata v0.1.0 +// from_xenium_to_spatialdata v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_xenium_to_spatialdata", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3373,7 +3373,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3425,12 +3425,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_spatialdata", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3455,7 +3455,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3927,7 +3927,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_xenium_to_spatialdata", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_xenium_to_spatialdata/nextflow.config b/target/nextflow/convert/from_xenium_to_spatialdata/nextflow.config index 50dd1c2..0b3e17f 100644 --- a/target/nextflow/convert/from_xenium_to_spatialdata/nextflow.config +++ b/target/nextflow/convert/from_xenium_to_spatialdata/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_xenium_to_spatialdata' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Converts the output from 10X Genomics Xenium dataset into a SpatialData objcet.\nBy default, the following files will be converted:\n - `experiment.xenium`: File containing specifications.\n - `nucleus_boundaries.parquet`: Polygons of nucleus boundaries.\n - `cell_boundaries.parquet`: Polygons of cell boundaries.\n - `transcripts.parquet`: File containing transcripts.\n - `cell_feature_matrix.h5`: File containing cell feature matrix.\n - `cells.parquet`: File containing cell metadata.\n - `morphology_mip.ome.tif`: File containing morphology mip.\n - `morphology_focus.ome.tif`: File containing morphology focus.\n \n' author = 'Dorien Roosen, Weiwei Schultz' } diff --git a/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml index 05c8490..1ed534a 100644 --- a/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "from_xenium_to_spatialexperiment" namespace: "convert" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -192,7 +192,7 @@ engines: id: "docker" image: "rocker/r2u:24.04" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -222,11 +222,11 @@ build_info: output: "target/nextflow/convert/from_xenium_to_spatialexperiment" executable: "target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -246,7 +246,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf b/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf index ef122fd..65bd6f3 100644 --- a/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf @@ -1,4 +1,4 @@ -// from_xenium_to_spatialexperiment v0.1.0 +// from_xenium_to_spatialexperiment v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "from_xenium_to_spatialexperiment", "namespace" : "convert", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3269,7 +3269,7 @@ meta = [ "id" : "docker", "image" : "rocker/r2u:24.04", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3311,12 +3311,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3341,7 +3341,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3801,7 +3801,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/convert/from_xenium_to_spatialexperiment", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/convert/from_xenium_to_spatialexperiment/nextflow.config b/target/nextflow/convert/from_xenium_to_spatialexperiment/nextflow.config index 9a71841..5987d5a 100644 --- a/target/nextflow/convert/from_xenium_to_spatialexperiment/nextflow.config +++ b/target/nextflow/convert/from_xenium_to_spatialexperiment/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'convert/from_xenium_to_spatialexperiment' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Creates a SpatialExperiment object from the downloaded unzipped Xenium Output Bundle directory\nfor 10x Genomics Xenium spatial gene expression data, and saves it as a SpatialExperiment object.\nThe constructor assumes the downloaded unzipped Xenium Output Bundle has the following structure:\n\nMandatory files\n· | — cell_feature_matrix.h5\n· | — cells.parquet\nOptional files, by default added to the metadata() as a list of paths (will be converted to parquet):\n· | — transcripts.parquet\n· | — cell_boundaries.parquet\n· | — nucleus_boundaries.parquet\n· | — experiment.xenium\n' author = 'Dorien Roosen' } diff --git a/target/nextflow/filter/subset_cosmx/.config.vsh.yaml b/target/nextflow/filter/subset_cosmx/.config.vsh.yaml index 6e596d0..93d8870 100644 --- a/target/nextflow/filter/subset_cosmx/.config.vsh.yaml +++ b/target/nextflow/filter/subset_cosmx/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "subset_cosmx" namespace: "filter" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dorien Roosen" roles: @@ -191,7 +191,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "apt" @@ -227,11 +227,11 @@ build_info: output: "target/nextflow/filter/subset_cosmx" executable: "target/nextflow/filter/subset_cosmx/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -251,7 +251,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/filter/subset_cosmx/main.nf b/target/nextflow/filter/subset_cosmx/main.nf index 1eb1bb0..37dfe98 100644 --- a/target/nextflow/filter/subset_cosmx/main.nf +++ b/target/nextflow/filter/subset_cosmx/main.nf @@ -1,4 +1,4 @@ -// subset_cosmx v0.1.0 +// subset_cosmx v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "subset_cosmx", "namespace" : "filter", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dorien Roosen", @@ -3281,7 +3281,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3333,12 +3333,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/filter/subset_cosmx", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3363,7 +3363,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3858,7 +3858,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/filter/subset_cosmx", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "label" : [ "lowmem", diff --git a/target/nextflow/filter/subset_cosmx/nextflow.config b/target/nextflow/filter/subset_cosmx/nextflow.config index 9ee0841..e77ee8b 100644 --- a/target/nextflow/filter/subset_cosmx/nextflow.config +++ b/target/nextflow/filter/subset_cosmx/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/subset_cosmx' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Filters the output from NanoString experiment to keep only a subset of the fields of view.\nExpected input folder structure:\npath/to/dataset/\n ├── CellComposite/\n ├── CellLabels/\n ├── CellOverlay/\n ├── CompartmentLabels/\n ├── _exprMat_file.csv\n ├── _fov_positions_file.csv\n ├── _metadata_file.csv\n └── _tx_file.csv \n' author = 'Dorien Roosen, Weiwei Schultz' } diff --git a/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml b/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml index 891b1e1..447fc44 100644 --- a/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml +++ b/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "spaceranger_count" namespace: "mapping" -version: "v0.1.0" +version: "v0.1" authors: - name: "Jakub Majercik" roles: @@ -408,7 +408,7 @@ engines: id: "docker" image: "ghcr.io/data-intuitive/spaceranger:3.1" target_registry: "images.viash-hub.com" - target_tag: "v0.1.0" + target_tag: "v0.1" namespace_separator: "/" setup: - type: "docker" @@ -426,11 +426,11 @@ build_info: output: "target/nextflow/mapping/spaceranger_count" executable: "target/nextflow/mapping/spaceranger_count/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -450,7 +450,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/mapping/spaceranger_count/main.nf b/target/nextflow/mapping/spaceranger_count/main.nf index d050355..0f043ae 100644 --- a/target/nextflow/mapping/spaceranger_count/main.nf +++ b/target/nextflow/mapping/spaceranger_count/main.nf @@ -1,4 +1,4 @@ -// spaceranger_count v0.1.0 +// spaceranger_count v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "spaceranger_count", "namespace" : "mapping", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Jakub Majercik", @@ -3526,7 +3526,7 @@ meta = [ "id" : "docker", "image" : "ghcr.io/data-intuitive/spaceranger:3.1", "target_registry" : "images.viash-hub.com", - "target_tag" : "v0.1.0", + "target_tag" : "v0.1", "namespace_separator" : "/", "setup" : [ { @@ -3548,12 +3548,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/mapping/spaceranger_count", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3578,7 +3578,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -4072,7 +4072,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline_spatial/mapping/spaceranger_count", - "tag" : "v0.1.0" + "tag" : "v0.1" }, "tag" : "$id" }'''), diff --git a/target/nextflow/mapping/spaceranger_count/nextflow.config b/target/nextflow/mapping/spaceranger_count/nextflow.config index d6b6de0..d2f8a1f 100644 --- a/target/nextflow/mapping/spaceranger_count/nextflow.config +++ b/target/nextflow/mapping/spaceranger_count/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'mapping/spaceranger_count' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'Count gene expression and protein expression reads from a single capture area.' author = 'Jakub Majercik' } diff --git a/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml b/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml index c03db62..61bf2d9 100644 --- a/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml +++ b/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "spatial_process_samples" namespace: "workflows/multiomics" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dries Schaumont" roles: @@ -555,18 +555,14 @@ dependencies: - name: "workflows/multiomics/process_samples" alias: "spatial_sample_processing" repository: - type: "github" - repo: "openpipelines-bio/openpipeline" - tag: "disable-scrublet_build" + type: "vsh" + repo: "openpipeline" + tag: "v3.0.0" repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" tag: "v3.0.0" -- type: "github" - name: "openpipeline_scrublet" - repo: "openpipelines-bio/openpipeline" - tag: "disable-scrublet_build" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -644,13 +640,13 @@ build_info: output: "target/nextflow/workflows/multiomics/spatial_process_samples" executable: "target/nextflow/workflows/multiomics/spatial_process_samples/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - - "target/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples" + - "target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -670,7 +666,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf b/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf index 8e64715..2d37919 100644 --- a/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf +++ b/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf @@ -1,4 +1,4 @@ -// spatial_process_samples v0.1.0 +// spatial_process_samples v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "spatial_process_samples", "namespace" : "workflows/multiomics", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dries Schaumont", @@ -3705,9 +3705,9 @@ meta = [ "name" : "workflows/multiomics/process_samples", "alias" : "spatial_sample_processing", "repository" : { - "type" : "github", - "repo" : "openpipelines-bio/openpipeline", - "tag" : "disable-scrublet_build" + "type" : "vsh", + "repo" : "openpipeline", + "tag" : "v3.0.0" } } ], @@ -3717,12 +3717,6 @@ meta = [ "name" : "openpipeline", "repo" : "openpipeline", "tag" : "v3.0.0" - }, - { - "type" : "github", - "name" : "openpipeline_scrublet", - "repo" : "openpipelines-bio/openpipeline", - "tag" : "disable-scrublet_build" } ], "links" : { @@ -3813,12 +3807,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/spatial_process_samples", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3843,7 +3837,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { @@ -3856,7 +3850,7 @@ meta = [ // resolve dependencies dependencies (if any) meta["root_dir"] = getRootDir() -include { process_samples as spatial_sample_processing_viashalias } from "${meta.root_dir}/dependencies/github/openpipelines-bio/openpipeline/disable-scrublet_build/nextflow/workflows/multiomics/process_samples/main.nf" +include { process_samples as spatial_sample_processing_viashalias } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/multiomics/process_samples/main.nf" spatial_sample_processing = spatial_sample_processing_viashalias.run(key: "spatial_sample_processing") // inner workflow @@ -3921,7 +3915,7 @@ workflow run_wf { "output": state.workflow_output ]}, args: [ - "skip_scrublet_filtering": "true", + "skip_scrublet_doublet_detection": "true", ], toState: [ "output": "output" diff --git a/target/nextflow/workflows/multiomics/spatial_process_samples/nextflow.config b/target/nextflow/workflows/multiomics/spatial_process_samples/nextflow.config index 31394ec..257f07f 100644 --- a/target/nextflow/workflows/multiomics/spatial_process_samples/nextflow.config +++ b/target/nextflow/workflows/multiomics/spatial_process_samples/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/spatial_process_samples' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'A pipeline to pre-process multiple spatial omics samples.' author = 'Dries Schaumont, Dorien Roosen, Weiwei Schultz' } diff --git a/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml b/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml index 9c63298..c23a8cd 100644 --- a/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml +++ b/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "spatial_qc" namespace: "workflows/qc" -version: "v0.1.0" +version: "v0.1" authors: - name: "Dries Schaumont" roles: @@ -387,13 +387,13 @@ build_info: output: "target/nextflow/workflows/qc/spatial_qc" executable: "target/nextflow/workflows/qc/spatial_qc/main.nf" viash_version: "0.9.4" - git_commit: "963107088819638e9e15d4f413c0a763dc81a192" + git_commit: "420b610b045c138f07475a1af39529f233d9af5a" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - "target/dependencies/vsh/vsh/openpipeline/v3.0.0/nextflow/workflows/qc/qc" package_config: name: "openpipeline_spatial" - version: "v0.1.0" + version: "v0.1" info: test_resources: - type: "s3" @@ -413,7 +413,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + - ".engines[.type == 'docker'].target_tag := 'v0.1'" organization: "vsh" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" diff --git a/target/nextflow/workflows/qc/spatial_qc/main.nf b/target/nextflow/workflows/qc/spatial_qc/main.nf index b0ae467..3347425 100644 --- a/target/nextflow/workflows/qc/spatial_qc/main.nf +++ b/target/nextflow/workflows/qc/spatial_qc/main.nf @@ -1,4 +1,4 @@ -// spatial_qc v0.1.0 +// spatial_qc v0.1 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "spatial_qc", "namespace" : "workflows/qc", - "version" : "v0.1.0", + "version" : "v0.1", "authors" : [ { "name" : "Dries Schaumont", @@ -3505,12 +3505,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/qc/spatial_qc", "viash_version" : "0.9.4", - "git_commit" : "963107088819638e9e15d4f413c0a763dc81a192", + "git_commit" : "420b610b045c138f07475a1af39529f233d9af5a", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { "name" : "openpipeline_spatial", - "version" : "v0.1.0", + "version" : "v0.1", "info" : { "test_resources" : [ { @@ -3535,7 +3535,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v0.1.0'" + ".engines[.type == 'docker'].target_tag := 'v0.1'" ], "organization" : "vsh", "links" : { diff --git a/target/nextflow/workflows/qc/spatial_qc/nextflow.config b/target/nextflow/workflows/qc/spatial_qc/nextflow.config index 50edaa1..cc844ac 100644 --- a/target/nextflow/workflows/qc/spatial_qc/nextflow.config +++ b/target/nextflow/workflows/qc/spatial_qc/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/qc/spatial_qc' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v0.1.0' + version = 'v0.1' description = 'A pipeline to add basic qc statistics to a MuData containing spatial data.' author = 'Dries Schaumont, Dorien Roosen, Weiwei Schultz' }