From adbd282fec1d68cf83623fd23604ff974cbc1f7f Mon Sep 17 00:00:00 2001 From: CI Date: Wed, 25 Mar 2026 11:11:31 +0000 Subject: [PATCH] Build branch openpipeline_spatial/niche-compass with version niche-compass to openpipeline_spatial on branch niche-compass (87e6260) Build pipeline: openpipelines-bio.openpipeline-spatial.niche-compass-8zcq9 Source commit: https://github.com/openpipelines-bio/openpipeline_spatial/commit/87e62605aafd706d539bf2978ef47ede6fe41926 Source message: add cmake to test container --- CHANGELOG.md | 18 +- _viash.yaml | 2 +- resources_test_scripts/visium_tiny.sh | 5 + resources_test_scripts/xenium_tiny.sh | 72 +- src/authors/luke_zappia.yaml | 12 + src/base/requirements/spatialdata.yaml | 3 + .../from_h5mu_to_spatialdata/config.vsh.yaml | 64 + .../from_h5mu_to_spatialdata/script.py | 60 + src/convert/from_h5mu_to_spatialdata/test.py | 143 + .../config.vsh.yaml | 1 + .../config.vsh.yaml | 2 - src/dataflow/split_h5mu/config.vsh.yaml | 73 - src/dataflow/split_h5mu/script.py | 119 - src/dataflow/split_h5mu/test.py | 288 -- .../spatial_autocorr/config.vsh.yaml | 111 + .../spatial_autocorr/script.py | 80 + .../spatial_autocorr/test.py | 220 + .../xenium_spatial_statistics/config.vsh.yaml | 109 + .../xenium_spatial_statistics/script.py | 315 ++ .../xenium_spatial_statistics/test.py | 75 + src/neighbors/join_graphs/config.vsh.yaml | 91 + src/neighbors/join_graphs/script.py | 63 + src/neighbors/join_graphs/test.py | 103 + src/nichecompass/nichecompass/config.vsh.yaml | 7 + src/nichecompass/nichecompass/script.py | 7 + src/nichecompass/nichecompass/test.py | 69 +- src/utils/subset_vars.py | 49 + .../niche/nichecompass_leiden/config.vsh.yaml | 13 +- .../niche/nichecompass_leiden/main.nf | 1 + .../niche/nichecompass_leiden/test.nf | 3 +- .../filter/subset_cosmx/.config.vsh.yaml | 6 +- .../filter/subset_cosmx/subset_cosmx | 4 +- .../filter/subset_cosmx/.config.vsh.yaml | 6 +- .../nextflow/filter/subset_cosmx/main.nf | 6 +- .../spaceranger_mapping_test/.config.vsh.yaml | 6 +- .../spaceranger_mapping_test | 4 +- .../nichecompass_leiden_test/.config.vsh.yaml | 6 +- .../nichecompass_leiden_test | 4 +- .../spaceranger_mapping_test/.config.vsh.yaml | 6 +- .../spaceranger_mapping_test/main.nf | 6 +- .../nichecompass_leiden_test/.config.vsh.yaml | 6 +- .../niche/nichecompass_leiden_test/main.nf | 6 +- .../split_modalities/.config.vsh.yaml | 8 +- .../multiomics/split_modalities/main.nf | 10 +- .../split_modalities/nextflow.config | 2 +- .../split_modalities/nextflow_labels.config | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../split_modalities/utils/labels.config | 0 .../split_modalities/utils/labels_ci.config | 0 .../rna/log_normalize/.config.vsh.yaml | 8 +- .../workflows/rna/log_normalize/main.nf | 10 +- .../rna/log_normalize/nextflow.config | 2 +- .../rna/log_normalize/nextflow_labels.config | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../rna/log_normalize/utils/labels.config | 0 .../rna/log_normalize/utils/labels_ci.config | 0 .../nextflow/cluster/leiden/.config.vsh.yaml | 10 +- .../nextflow/cluster/leiden/compress_h5mu.py | 0 .../nextflow/cluster/leiden/main.nf | 14 +- .../nextflow/cluster/leiden/nextflow.config | 2 +- .../cluster/leiden/nextflow_labels.config | 0 .../cluster/leiden/nextflow_schema.json | 0 .../nextflow/cluster/leiden/setup_logger.py | 0 .../concatenate_h5mu/.config.vsh.yaml | 10 +- .../concatenate_h5mu/compress_h5mu.py | 0 .../dataflow/concatenate_h5mu/main.nf | 14 +- .../dataflow/concatenate_h5mu/nextflow.config | 2 +- .../concatenate_h5mu/nextflow_labels.config | 0 .../concatenate_h5mu/nextflow_schema.json | 0 .../dataflow/concatenate_h5mu/setup_logger.py | 0 .../nextflow/dataflow/merge/.config.vsh.yaml | 10 +- .../nextflow/dataflow/merge/main.nf | 14 +- .../nextflow/dataflow/merge/nextflow.config | 2 +- .../dataflow/merge/nextflow_labels.config | 0 .../dataflow/merge/nextflow_schema.json | 0 .../nextflow/dataflow/merge/setup_logger.py | 0 .../dataflow/split_h5mu/.config.vsh.yaml | 55 +- .../nextflow/dataflow/split_h5mu/main.nf | 63 +- .../dataflow/split_h5mu/nextflow.config | 2 +- .../split_h5mu}/nextflow_labels.config | 0 .../dataflow/split_h5mu/nextflow_schema.json | 0 .../dataflow/split_h5mu}/setup_logger.py | 0 .../split_modalities/.config.vsh.yaml | 10 +- .../dataflow/split_modalities/main.nf | 14 +- .../dataflow/split_modalities/nextflow.config | 2 +- .../split_modalities}/nextflow_labels.config | 0 .../split_modalities/nextflow_schema.json | 0 .../split_modalities}/setup_logger.py | 0 .../nextflow/dimred/pca/.config.vsh.yaml | 10 +- .../nextflow/dimred/pca/compress_h5mu.py | 0 .../nextflow/dimred/pca/main.nf | 14 +- .../nextflow/dimred/pca/nextflow.config | 2 +- .../dimred/pca}/nextflow_labels.config | 0 .../nextflow/dimred/pca/nextflow_schema.json | 0 .../nextflow/dimred/pca}/setup_logger.py | 0 .../nextflow/dimred/umap/.config.vsh.yaml | 10 +- .../nextflow/dimred/umap/compress_h5mu.py | 0 .../nextflow/dimred/umap/main.nf | 14 +- .../nextflow/dimred/umap/nextflow.config | 2 +- .../dimred/umap}/nextflow_labels.config | 0 .../nextflow/dimred/umap/nextflow_schema.json | 0 .../nextflow/dimred/umap}/setup_logger.py | 0 .../.config.vsh.yaml | 10 +- .../compress_h5mu.py | 0 .../highly_variable_features_scanpy/main.nf | 14 +- .../nextflow.config | 2 +- .../nextflow_labels.config | 0 .../nextflow_schema.json | 0 .../setup_logger.py | 0 .../subset_vars.py | 0 .../filter/delimit_fraction/.config.vsh.yaml | 10 +- .../filter/delimit_fraction/compress_h5mu.py | 0 .../nextflow/filter/delimit_fraction/main.nf | 14 +- .../filter/delimit_fraction/nextflow.config | 2 +- .../delimit_fraction}/nextflow_labels.config | 0 .../delimit_fraction/nextflow_schema.json | 0 .../filter/delimit_fraction}/setup_logger.py | 0 .../filter/do_filter/.config.vsh.yaml | 10 +- .../filter/do_filter/compress_h5mu.py | 0 .../nextflow/filter/do_filter/main.nf | 14 +- .../nextflow/filter/do_filter/nextflow.config | 2 +- .../filter/do_filter}/nextflow_labels.config | 0 .../filter/do_filter/nextflow_schema.json | 0 .../filter/do_filter}/setup_logger.py | 0 .../filter_with_counts/.config.vsh.yaml | 10 +- .../filter_with_counts/compress_h5mu.py | 0 .../filter/filter_with_counts/main.nf | 14 +- .../filter/filter_with_counts/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../filter_with_counts/nextflow_schema.json | 0 .../filter_with_counts}/setup_logger.py | 0 .../filter_with_scrublet/.config.vsh.yaml | 10 +- .../filter_with_scrublet/compress_h5mu.py | 0 .../filter/filter_with_scrublet/main.nf | 14 +- .../filter_with_scrublet/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../filter_with_scrublet/nextflow_schema.json | 0 .../filter_with_scrublet}/setup_logger.py | 0 .../nextflow/metadata/add_id/.config.vsh.yaml | 10 +- .../nextflow/metadata/add_id/main.nf | 14 +- .../nextflow/metadata/add_id/nextflow.config | 2 +- .../metadata/add_id}/nextflow_labels.config | 0 .../metadata/add_id/nextflow_schema.json | 0 .../nextflow/metadata/add_id}/setup_logger.py | 0 .../grep_annotation_column/.config.vsh.yaml | 10 +- .../grep_annotation_column/compress_h5mu.py | 0 .../metadata/grep_annotation_column/main.nf | 14 +- .../grep_annotation_column/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../nextflow_schema.json | 0 .../grep_annotation_column}/setup_logger.py | 0 .../move_obsm_to_obs/.config.vsh.yaml | 10 +- .../move_obsm_to_obs/compress_h5mu.py | 0 .../metadata/move_obsm_to_obs/main.nf | 14 +- .../metadata/move_obsm_to_obs/nextflow.config | 2 +- .../move_obsm_to_obs}/nextflow_labels.config | 0 .../move_obsm_to_obs/nextflow_schema.json | 0 .../move_obsm_to_obs}/setup_logger.py | 0 .../neighbors/find_neighbors/.config.vsh.yaml | 10 +- .../neighbors/find_neighbors/compress_h5mu.py | 0 .../nextflow/neighbors/find_neighbors/main.nf | 14 +- .../neighbors/find_neighbors/nextflow.config | 2 +- .../find_neighbors}/nextflow_labels.config | 0 .../find_neighbors/nextflow_schema.json | 0 .../neighbors/find_neighbors}/setup_logger.py | 0 .../qc/calculate_qc_metrics/.config.vsh.yaml | 10 +- .../qc/calculate_qc_metrics/compress_h5mu.py | 0 .../nextflow/qc/calculate_qc_metrics/main.nf | 14 +- .../qc/calculate_qc_metrics/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../calculate_qc_metrics/nextflow_schema.json | 0 .../qc/calculate_qc_metrics}/setup_logger.py | 0 .../nextflow/transform/clr/.config.vsh.yaml | 10 +- .../nextflow/transform/clr/compress_h5mu.py | 0 .../nextflow/transform/clr/main.nf | 14 +- .../nextflow/transform/clr/nextflow.config | 2 +- .../transform/clr}/nextflow_labels.config | 0 .../transform/clr/nextflow_schema.json | 0 .../transform/delete_layer/.config.vsh.yaml | 10 +- .../transform/delete_layer/compress_h5mu.py | 0 .../nextflow/transform/delete_layer/main.nf | 14 +- .../transform/delete_layer/nextflow.config | 2 +- .../delete_layer}/nextflow_labels.config | 0 .../delete_layer/nextflow_schema.json | 0 .../transform/delete_layer}/setup_logger.py | 0 .../nextflow/transform/log1p/.config.vsh.yaml | 10 +- .../nextflow/transform/log1p/compress_h5mu.py | 0 .../nextflow/transform/log1p/main.nf | 14 +- .../nextflow/transform/log1p/nextflow.config | 2 +- .../transform/log1p}/nextflow_labels.config | 0 .../transform/log1p/nextflow_schema.json | 0 .../nextflow/transform/log1p}/setup_logger.py | 0 .../normalize_total/.config.vsh.yaml | 10 +- .../normalize_total/compress_h5mu.py | 0 .../transform/normalize_total/main.nf | 14 +- .../transform/normalize_total/nextflow.config | 2 +- .../normalize_total}/nextflow_labels.config | 0 .../normalize_total/nextflow_schema.json | 0 .../normalize_total}/setup_logger.py | 0 .../nextflow/transform/scale/.config.vsh.yaml | 10 +- .../nextflow/transform/scale/compress_h5mu.py | 0 .../nextflow/transform/scale/main.nf | 14 +- .../nextflow/transform/scale/nextflow.config | 2 +- .../transform/scale}/nextflow_labels.config | 0 .../transform/scale/nextflow_schema.json | 0 .../nextflow/transform/scale}/setup_logger.py | 0 .../gdo/gdo_singlesample/.config.vsh.yaml | 8 +- .../workflows/gdo/gdo_singlesample/main.nf | 10 +- .../gdo/gdo_singlesample/nextflow.config | 2 +- .../gdo_singlesample}/nextflow_labels.config | 0 .../gdo/gdo_singlesample/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../gdo/gdo_singlesample/utils/labels.config | 0 .../gdo_singlesample/utils/labels_ci.config | 0 .../dimensionality_reduction/.config.vsh.yaml | 8 +- .../dimensionality_reduction/main.nf | 10 +- .../dimensionality_reduction/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../utils/labels.config | 0 .../utils/labels_ci.config | 0 .../neighbors_leiden_umap/.config.vsh.yaml | 8 +- .../multiomics/neighbors_leiden_umap/main.nf | 10 +- .../neighbors_leiden_umap/nextflow.config | 2 +- .../nextflow_labels.config | 0 .../nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../neighbors_leiden_umap/utils/labels.config | 0 .../utils/labels_ci.config | 0 .../process_batches/.config.vsh.yaml | 8 +- .../multiomics/process_batches/main.nf | 10 +- .../process_batches/nextflow.config | 2 +- .../process_batches}/nextflow_labels.config | 0 .../process_batches/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../process_batches/utils/labels.config | 0 .../process_batches/utils/labels_ci.config | 0 .../process_samples/.config.vsh.yaml | 8 +- .../multiomics/process_samples/main.nf | 10 +- .../process_samples/nextflow.config | 2 +- .../process_samples}/nextflow_labels.config | 0 .../process_samples/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../process_samples/utils/labels.config | 0 .../process_samples/utils/labels_ci.config | 0 .../prot/prot_multisample/.config.vsh.yaml | 8 +- .../workflows/prot/prot_multisample/main.nf | 10 +- .../prot/prot_multisample/nextflow.config | 2 +- .../prot_multisample}/nextflow_labels.config | 0 .../prot_multisample/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../prot/prot_multisample/utils/labels.config | 0 .../prot_multisample/utils/labels_ci.config | 0 .../prot/prot_singlesample/.config.vsh.yaml | 8 +- .../workflows/prot/prot_singlesample/main.nf | 10 +- .../prot/prot_singlesample/nextflow.config | 2 +- .../prot_singlesample}/nextflow_labels.config | 0 .../prot_singlesample/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../prot_singlesample/utils/labels.config | 0 .../prot_singlesample/utils/labels_ci.config | 0 .../nextflow/workflows/qc/qc/.config.vsh.yaml | 8 +- .../nextflow/workflows/qc/qc/main.nf | 10 +- .../nextflow/workflows/qc/qc/nextflow.config | 2 +- .../workflows/qc/qc}/nextflow_labels.config | 0 .../workflows/qc/qc/nextflow_schema.json | 0 .../qc/qc/utils/errorstrat_ignore.config | 0 .../qc/qc/utils/integration_tests.config | 0 .../workflows/qc/qc/utils/labels.config | 0 .../workflows/qc/qc/utils/labels_ci.config | 0 .../rna/rna_multisample/.config.vsh.yaml | 8 +- .../workflows/rna/rna_multisample/main.nf | 10 +- .../rna/rna_multisample/nextflow.config | 2 +- .../rna_multisample}/nextflow_labels.config | 0 .../rna/rna_multisample/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../rna/rna_multisample/utils/labels.config | 0 .../rna_multisample/utils/labels_ci.config | 0 .../rna/rna_singlesample/.config.vsh.yaml | 8 +- .../workflows/rna/rna_singlesample/main.nf | 10 +- .../rna/rna_singlesample/nextflow.config | 2 +- .../rna_singlesample/nextflow_labels.config} | 0 .../rna/rna_singlesample/nextflow_schema.json | 0 .../utils/errorstrat_ignore.config | 0 .../utils/integration_tests.config | 0 .../rna/rna_singlesample/utils/labels.config | 48 + .../rna_singlesample/utils/labels_ci.config | 0 .../from_cells2stats_to_h5mu/.config.vsh.yaml | 6 +- .../from_cells2stats_to_h5mu | 4 +- .../from_cosmx_to_h5mu/.config.vsh.yaml | 6 +- .../from_cosmx_to_h5mu/from_cosmx_to_h5mu | 4 +- .../.config.vsh.yaml | 6 +- .../from_cosmx_to_spatialexperiment | 4 +- .../.config.vsh.yaml | 131 +- .../from_h5mu_to_spatialdata | 1234 +++++ .../nextflow_labels.config | 0 .../from_h5mu_to_spatialdata}/setup_logger.py | 0 .../.config.vsh.yaml | 7 +- .../from_h5mu_to_spatialexperiment | 4 +- .../from_spaceranger_to_h5mu/.config.vsh.yaml | 6 +- .../from_spaceranger_to_h5mu | 4 +- .../from_spatialdata_to_h5mu/.config.vsh.yaml | 7 +- .../from_spatialdata_to_h5mu | 6 +- .../from_xenium_to_h5mu/.config.vsh.yaml | 6 +- .../from_xenium_to_h5mu/from_xenium_to_h5mu | 4 +- .../.config.vsh.yaml | 11 +- .../from_xenium_to_spatialdata | 9 +- .../.config.vsh.yaml | 12 +- .../from_xenium_to_spatialexperiment | 6 +- .../spatial_autocorr/.config.vsh.yaml | 309 ++ .../spatial_autocorr}/nextflow_labels.config | 0 .../spatial_autocorr/spatial_autocorr | 1421 ++++++ .../.config.vsh.yaml | 279 ++ .../nextflow_labels.config | 68 + .../xenium_spatial_statistics | 1571 ++++++ .../spaceranger_count/.config.vsh.yaml | 6 +- .../spaceranger_count/spaceranger_count | 4 +- .../neighbors/join_graphs/.config.vsh.yaml | 295 ++ .../join_graphs/join_graphs} | 332 +- .../join_graphs/nextflow_labels.config | 68 + .../neighbors/join_graphs/setup_logger.py | 12 + .../.config.vsh.yaml | 6 +- .../spatial_neighborhood_graph | 4 +- .../nichecompass/.config.vsh.yaml | 19 +- .../nichecompass/nichecompass/nichecompass | 32 +- .../nichecompass/nichecompass/subset_vars.py | 49 + .../from_cells2stats_to_h5mu/.config.vsh.yaml | 6 +- .../convert/from_cells2stats_to_h5mu/main.nf | 6 +- .../from_cosmx_to_h5mu/.config.vsh.yaml | 6 +- .../convert/from_cosmx_to_h5mu/main.nf | 6 +- .../.config.vsh.yaml | 6 +- .../from_cosmx_to_spatialexperiment/main.nf | 6 +- .../from_h5mu_to_spatialdata/.config.vsh.yaml | 268 ++ .../convert/from_h5mu_to_spatialdata/main.nf | 3986 ++++++++++++++++ .../from_h5mu_to_spatialdata/nextflow.config | 126 + .../nextflow_labels.config | 68 + .../nextflow_schema.json | 61 + .../from_h5mu_to_spatialdata/setup_logger.py | 12 + .../.config.vsh.yaml | 7 +- .../from_h5mu_to_spatialexperiment/main.nf | 9 +- .../from_spaceranger_to_h5mu/.config.vsh.yaml | 6 +- .../convert/from_spaceranger_to_h5mu/main.nf | 6 +- .../from_spatialdata_to_h5mu/.config.vsh.yaml | 7 +- .../convert/from_spatialdata_to_h5mu/main.nf | 7 +- .../from_xenium_to_h5mu/.config.vsh.yaml | 6 +- .../convert/from_xenium_to_h5mu/main.nf | 6 +- .../.config.vsh.yaml | 11 +- .../from_xenium_to_spatialdata/main.nf | 10 +- .../.config.vsh.yaml | 12 +- .../from_xenium_to_spatialexperiment/main.nf | 14 +- .../spatial_autocorr/.config.vsh.yaml | 309 ++ .../spatial_autocorr/main.nf | 4030 ++++++++++++++++ .../spatial_autocorr/nextflow.config | 125 + .../spatial_autocorr/nextflow_labels.config | 68 + .../spatial_autocorr/nextflow_schema.json | 133 + .../.config.vsh.yaml | 279 ++ .../xenium_spatial_statistics/main.nf | 4234 +++++++++++++++++ .../xenium_spatial_statistics/nextflow.config | 125 + .../nextflow_labels.config | 68 + .../nextflow_schema.json | 105 + .../spaceranger_count/.config.vsh.yaml | 6 +- .../mapping/spaceranger_count/main.nf | 6 +- .../neighbors/join_graphs/.config.vsh.yaml | 295 ++ target/nextflow/neighbors/join_graphs/main.nf | 4012 ++++++++++++++++ .../neighbors/join_graphs/nextflow.config | 126 + .../join_graphs/nextflow_labels.config | 68 + .../join_graphs/nextflow_schema.json | 108 + .../neighbors/join_graphs/setup_logger.py | 12 + .../.config.vsh.yaml | 6 +- .../spatial_neighborhood_graph/main.nf | 6 +- .../nichecompass/.config.vsh.yaml | 19 +- .../nichecompass/nichecompass/main.nf | 29 +- .../nichecompass/nichecompass/subset_vars.py | 49 + .../spaceranger_mapping/.config.vsh.yaml | 6 +- .../ingestion/spaceranger_mapping/main.nf | 6 +- .../spatial_process_samples/.config.vsh.yaml | 10 +- .../spatial_process_samples/main.nf | 10 +- .../nichecompass_leiden/.config.vsh.yaml | 38 +- .../niche/nichecompass_leiden/main.nf | 45 +- .../nichecompass_leiden/nextflow_schema.json | 12 +- .../workflows/qc/spatial_qc/.config.vsh.yaml | 10 +- .../nextflow/workflows/qc/spatial_qc/main.nf | 10 +- 393 files changed, 26718 insertions(+), 1455 deletions(-) mode change 100644 => 100755 resources_test_scripts/visium_tiny.sh create mode 100644 src/authors/luke_zappia.yaml create mode 100644 src/convert/from_h5mu_to_spatialdata/config.vsh.yaml create mode 100644 src/convert/from_h5mu_to_spatialdata/script.py create mode 100644 src/convert/from_h5mu_to_spatialdata/test.py delete mode 100644 src/dataflow/split_h5mu/config.vsh.yaml delete mode 100644 src/dataflow/split_h5mu/script.py delete mode 100644 src/dataflow/split_h5mu/test.py create mode 100644 src/feature_annotation/spatial_autocorr/config.vsh.yaml create mode 100644 src/feature_annotation/spatial_autocorr/script.py create mode 100644 src/feature_annotation/spatial_autocorr/test.py create mode 100644 src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml create mode 100644 src/feature_annotation/xenium_spatial_statistics/script.py create mode 100644 src/feature_annotation/xenium_spatial_statistics/test.py create mode 100644 src/neighbors/join_graphs/config.vsh.yaml create mode 100644 src/neighbors/join_graphs/script.py create mode 100644 src/neighbors/join_graphs/test.py create mode 100644 src/utils/subset_vars.py rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/_private/nextflow/workflows/rna/log_normalize/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/cluster/leiden/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/concatenate_h5mu/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/merge/setup_logger.py (100%) rename target/{ => dependencies/vsh/vsh/openpipeline/v4.0.3}/nextflow/dataflow/split_h5mu/.config.vsh.yaml (81%) rename target/{ => dependencies/vsh/vsh/openpipeline/v4.0.3}/nextflow/dataflow/split_h5mu/main.nf (98%) rename target/{ => dependencies/vsh/vsh/openpipeline/v4.0.3}/nextflow/dataflow/split_h5mu/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dataflow/split_modalities => v4.0.3/nextflow/dataflow/split_h5mu}/nextflow_labels.config (100%) rename target/{ => dependencies/vsh/vsh/openpipeline/v4.0.3}/nextflow/dataflow/split_h5mu/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dataflow/split_modalities => v4.0.3/nextflow/dataflow/split_h5mu}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/split_modalities/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/split_modalities/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/split_modalities/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dimred/pca => v4.0.3/nextflow/dataflow/split_modalities}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dataflow/split_modalities/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dimred/pca => v4.0.3/nextflow/dataflow/split_modalities}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/pca/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/pca/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/pca/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/pca/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dimred/umap => v4.0.3/nextflow/dimred/pca}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/pca/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/dimred/umap => v4.0.3/nextflow/dimred/pca}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/umap/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/umap/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/umap/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/umap/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy => v4.0.3/nextflow/dimred/umap}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/dimred/umap/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy => v4.0.3/nextflow/dimred/umap}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/delimit_fraction => v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/delimit_fraction => v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/delimit_fraction/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/delimit_fraction/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/delimit_fraction/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/delimit_fraction/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/do_filter => v4.0.3/nextflow/filter/delimit_fraction}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/delimit_fraction/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/do_filter => v4.0.3/nextflow/filter/delimit_fraction}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/do_filter/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/do_filter/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/do_filter/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/do_filter/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/filter_with_counts => v4.0.3/nextflow/filter/do_filter}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/do_filter/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/filter_with_counts => v4.0.3/nextflow/filter/do_filter}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_counts/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_counts/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_counts/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_counts/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/filter_with_scrublet => v4.0.3/nextflow/filter/filter_with_counts}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_counts/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/filter/filter_with_scrublet => v4.0.3/nextflow/filter/filter_with_counts}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_scrublet/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_scrublet/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_scrublet/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_scrublet/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/add_id => v4.0.3/nextflow/filter/filter_with_scrublet}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/filter/filter_with_scrublet/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/add_id => v4.0.3/nextflow/filter/filter_with_scrublet}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/add_id/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/add_id/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/add_id/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/grep_annotation_column => v4.0.3/nextflow/metadata/add_id}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/add_id/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/grep_annotation_column => v4.0.3/nextflow/metadata/add_id}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/grep_annotation_column/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/grep_annotation_column/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/grep_annotation_column/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/grep_annotation_column/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/move_obsm_to_obs => v4.0.3/nextflow/metadata/grep_annotation_column}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/grep_annotation_column/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/metadata/move_obsm_to_obs => v4.0.3/nextflow/metadata/grep_annotation_column}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/move_obsm_to_obs/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/move_obsm_to_obs/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/neighbors/find_neighbors => v4.0.3/nextflow/metadata/move_obsm_to_obs}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/neighbors/find_neighbors => v4.0.3/nextflow/metadata/move_obsm_to_obs}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/neighbors/find_neighbors/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/neighbors/find_neighbors/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/neighbors/find_neighbors/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/neighbors/find_neighbors/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/qc/calculate_qc_metrics => v4.0.3/nextflow/neighbors/find_neighbors}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/neighbors/find_neighbors/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/qc/calculate_qc_metrics => v4.0.3/nextflow/neighbors/find_neighbors}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/qc/calculate_qc_metrics/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/qc/calculate_qc_metrics/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/qc/calculate_qc_metrics/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/clr => v4.0.3/nextflow/qc/calculate_qc_metrics}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/qc/calculate_qc_metrics/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/delete_layer => v4.0.3/nextflow/qc/calculate_qc_metrics}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/clr/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/clr/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/clr/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/clr/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/delete_layer => v4.0.3/nextflow/transform/clr}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/clr/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/delete_layer/.config.vsh.yaml (97%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/delete_layer/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/delete_layer/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/delete_layer/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/log1p => v4.0.3/nextflow/transform/delete_layer}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/delete_layer/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/log1p => v4.0.3/nextflow/transform/delete_layer}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/log1p/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/log1p/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/log1p/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/log1p/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/normalize_total => v4.0.3/nextflow/transform/log1p}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/log1p/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/normalize_total => v4.0.3/nextflow/transform/log1p}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/normalize_total/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/normalize_total/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/normalize_total/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/normalize_total/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/scale => v4.0.3/nextflow/transform/normalize_total}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/normalize_total/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/transform/scale => v4.0.3/nextflow/transform/normalize_total}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/scale/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/scale/compress_h5mu.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/scale/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/scale/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/gdo/gdo_singlesample => v4.0.3/nextflow/transform/scale}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/transform/scale/nextflow_schema.json (100%) rename target/{executable/dataflow/split_h5mu => dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale}/setup_logger.py (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction => v4.0.3/nextflow/workflows/gdo/gdo_singlesample}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap => v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/multiomics/process_batches => v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/multiomics/process_samples => v4.0.3/nextflow/workflows/multiomics/process_batches}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/prot/prot_multisample => v4.0.3/nextflow/workflows/multiomics/process_samples}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/prot/prot_singlesample => v4.0.3/nextflow/workflows/prot/prot_multisample}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/qc/qc => v4.0.3/nextflow/workflows/prot/prot_singlesample}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/rna/rna_multisample => v4.0.3/nextflow/workflows/qc/qc}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/qc/qc/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/rna/rna_singlesample => v4.0.3/nextflow/workflows/rna/rna_multisample}/nextflow_labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/utils/labels.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml (98%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/main.nf (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/nextflow.config (99%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/labels.config => v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config} (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config (100%) rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config (100%) create mode 100644 target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels.config rename target/dependencies/vsh/vsh/openpipeline/{v4.0.2 => v4.0.3}/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config (100%) rename target/executable/{dataflow/split_h5mu => convert/from_h5mu_to_spatialdata}/.config.vsh.yaml (75%) create mode 100755 target/executable/convert/from_h5mu_to_spatialdata/from_h5mu_to_spatialdata rename target/executable/{dataflow/split_h5mu => convert/from_h5mu_to_spatialdata}/nextflow_labels.config (100%) rename target/{nextflow/dataflow/split_h5mu => executable/convert/from_h5mu_to_spatialdata}/setup_logger.py (100%) create mode 100644 target/executable/feature_annotation/spatial_autocorr/.config.vsh.yaml rename target/{nextflow/dataflow/split_h5mu => executable/feature_annotation/spatial_autocorr}/nextflow_labels.config (100%) create mode 100755 target/executable/feature_annotation/spatial_autocorr/spatial_autocorr create mode 100644 target/executable/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml create mode 100644 target/executable/feature_annotation/xenium_spatial_statistics/nextflow_labels.config create mode 100755 target/executable/feature_annotation/xenium_spatial_statistics/xenium_spatial_statistics create mode 100644 target/executable/neighbors/join_graphs/.config.vsh.yaml rename target/executable/{dataflow/split_h5mu/split_h5mu => neighbors/join_graphs/join_graphs} (82%) create mode 100644 target/executable/neighbors/join_graphs/nextflow_labels.config create mode 100644 target/executable/neighbors/join_graphs/setup_logger.py create mode 100644 target/executable/nichecompass/nichecompass/subset_vars.py create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/.config.vsh.yaml create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/main.nf create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/nextflow.config create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_labels.config create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_schema.json create mode 100644 target/nextflow/convert/from_h5mu_to_spatialdata/setup_logger.py create mode 100644 target/nextflow/feature_annotation/spatial_autocorr/.config.vsh.yaml create mode 100644 target/nextflow/feature_annotation/spatial_autocorr/main.nf create mode 100644 target/nextflow/feature_annotation/spatial_autocorr/nextflow.config create mode 100644 target/nextflow/feature_annotation/spatial_autocorr/nextflow_labels.config create mode 100644 target/nextflow/feature_annotation/spatial_autocorr/nextflow_schema.json create mode 100644 target/nextflow/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml create mode 100644 target/nextflow/feature_annotation/xenium_spatial_statistics/main.nf create mode 100644 target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow.config create mode 100644 target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_labels.config create mode 100644 target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_schema.json create mode 100644 target/nextflow/neighbors/join_graphs/.config.vsh.yaml create mode 100644 target/nextflow/neighbors/join_graphs/main.nf create mode 100644 target/nextflow/neighbors/join_graphs/nextflow.config create mode 100644 target/nextflow/neighbors/join_graphs/nextflow_labels.config create mode 100644 target/nextflow/neighbors/join_graphs/nextflow_schema.json create mode 100644 target/nextflow/neighbors/join_graphs/setup_logger.py create mode 100644 target/nextflow/nichecompass/nichecompass/subset_vars.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 615834e..2d72d6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,10 +1,24 @@ -# openpipeline_spatial x.x.x +# openpipeline_spatial 0.4.0 + +## NEW FUNCTIONALITY + +* `feature_annotation/spatial_autocorr`: Added a spatial autocorrelation component (PR #45). + +* `feature_annotation/xenium_spatial_statistics`: Added a component to calculate spatial statistics on Xenium-ingested data (PR #46). + +* `neighbors/join_graphs`: Combine spatial and expression neighborhood graphs into a single graph (PR #47). + +# openpipeline_spatial 0.3.0 + +## NEW FUNCTIONALITY + +* `convert/from_h5mu_to_spatialdata`: Added a converter component to convert from H5MU to SpatialData (PR #40) ## MINOR CHANGES * Bump squidpy to 1.8.1 and spatialdata to 0.7.2 (PR #41). -* Update openpipeline dependencies to v4.0.2 (PR #42). +* Update openpipeline dependencies to v4.0.3 (PR #42, PR #44). # openpipeline_spatial 0.2.0 diff --git a/_viash.yaml b/_viash.yaml index 7db3b34..f818c47 100644 --- a/_viash.yaml +++ b/_viash.yaml @@ -10,7 +10,7 @@ repositories: - name: openpipeline repo: openpipeline type: vsh - tag: v4.0.2 + tag: v4.0.3 info: test_resources: - type: s3 diff --git a/resources_test_scripts/visium_tiny.sh b/resources_test_scripts/visium_tiny.sh old mode 100644 new mode 100755 index 5bc46f6..db105d3 --- a/resources_test_scripts/visium_tiny.sh +++ b/resources_test_scripts/visium_tiny.sh @@ -7,6 +7,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel) # Define absolute directory path DIR="$REPO_ROOT/resources_test/visium" +ID="Visium_FFPE_Human_Ovarian_Cancer_tiny" # from https://www.10xgenomics.com/resources/datasets/human-ovarian-cancer-1-standard mkdir -p "$DIR" @@ -46,6 +47,10 @@ echo "> Running spaceranger complete" rm -rf "$DIR/Visium_FFPE_Human_Ovarian_Cancer_fastqs" rm -f "$DIR/Visium_FFPE_Human_Ovarian_Cancer_image.jpg" +viash run "$REPO_ROOT/src/convert/from_spaceranger_to_h5mu/config.vsh.yaml" -- \ + --input "$DIR/Visium_FFPE_Human_Ovarian_Cancer_tiny_spaceranger" \ + --output "$DIR/$ID.h5mu" + aws s3 sync \ --profile di \ --exclude "*.yaml" \ diff --git a/resources_test_scripts/xenium_tiny.sh b/resources_test_scripts/xenium_tiny.sh index 1e927a1..f00d1a4 100755 --- a/resources_test_scripts/xenium_tiny.sh +++ b/resources_test_scripts/xenium_tiny.sh @@ -19,14 +19,16 @@ function clean_up { trap clean_up EXIT if [ ! -d "$OUT" ]; then - tiny_dataset="https://raw.githubusercontent.com/nf-core/test-datasets/spatialxe/Xenium_Prime_Mouse_Ileum_tiny_outs.zip" - wget "$tiny_dataset" -O "$TMPDIR/xenium_tiny.zip" + tiny_dataset="https://raw.githubusercontent.com/nf-core/test-datasets/spatialxe/Xenium_Prime_Mouse_Ileum_tiny_outs.tar.gz" + wget "$tiny_dataset" -O "$TMPDIR/xenium_tiny.tar.gz" - unzip -q "$TMPDIR/xenium_tiny.zip" -d "$TMPDIR/xenium_tiny" + mkdir -p "$TMPDIR/xenium_tiny" + tar -xzf "$TMPDIR/xenium_tiny.tar.gz" -C "$TMPDIR/xenium_tiny" mkdir -p "$OUT" mv "$TMPDIR/xenium_tiny/Xenium_Prime_Mouse_Ileum_tiny_outs/"* "$OUT/" fi +rm -rf "$DIR/$ID.zarr" viash run "$REPO_ROOT/src/convert/from_xenium_to_spatialdata/config.vsh.yaml" -- \ --input "$OUT" \ --output "$DIR/$ID.zarr" @@ -35,11 +37,73 @@ viash run "$REPO_ROOT/src/convert/from_spatialdata_to_h5mu/config.vsh.yaml" -- \ --input "$DIR/$ID.zarr" \ --output "$DIR/$ID.h5mu" - viash run src/neighbors/spatial_neighborhood_graph/config.vsh.yaml -- \ --input "$DIR/$ID.h5mu" \ --output "$DIR/$ID.h5mu" +cat > /tmp/qc.yaml < /tmp/pca.yaml < /tmp/find_neighbors.yaml < 2") diff --git a/src/convert/from_h5mu_to_spatialdata/config.vsh.yaml b/src/convert/from_h5mu_to_spatialdata/config.vsh.yaml new file mode 100644 index 0000000..1c36718 --- /dev/null +++ b/src/convert/from_h5mu_to_spatialdata/config.vsh.yaml @@ -0,0 +1,64 @@ +name: "from_h5mu_to_spatialdata" +namespace: "convert" +scope: "public" +description: | + Reads in an H5MU file and saves it as a SpatialData Zarr store. The selected + modality in the MuData is stored as the main table in the SpatialData object. + If a matching existing SpatialData is provided, it will be used to fill the + remaining SpatialData slots. + +authors: + - __merge__: /src/authors/dorien_roosen.yaml + roles: [ maintainer ] + - __merge__: /src/authors/luke_zappia.yaml + roles: [ author ] +arguments: + - name: "--input" + alternatives: ["-i"] + type: file + description: Input H5MU file. + example: input.h5mu + direction: input + required: true + - name: "--input_spatialdata" + type: file + description: An optional existing SpatialData Zarr store to fill remaining slots from. + example: existing.zarr + direction: input + required: false + - name: "--modality" + type: string + default: rna + description: The modality in the MuData to be used as the main table in the SpatialData object. + - name: "--output" + alternatives: ["-o"] + type: file + description: The path to the output SpatialData Zarr store. + example: "output.zarr" + direction: output + required: true + +resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py +test_resources: + - type: python_script + path: test.py + - path: /resources_test/xenium/xenium_tiny.h5mu + - path: /resources_test/xenium/xenium_tiny.zarr +engines: + - type: docker + image: python:3.12-slim + setup: + - type: apt + packages: + - procps + - type: python + __merge__: [/src/base/requirements/anndata_mudata.yaml, /src/base/requirements/spatialdata.yaml] + __merge__: [ /src/base/requirements/python_test_setup.yaml, .] +runners: + - type: executable + - type: nextflow + directives: + label: [lowmem, singlecpu] diff --git a/src/convert/from_h5mu_to_spatialdata/script.py b/src/convert/from_h5mu_to_spatialdata/script.py new file mode 100644 index 0000000..fc793bb --- /dev/null +++ b/src/convert/from_h5mu_to_spatialdata/script.py @@ -0,0 +1,60 @@ +import logging +import sys + +import mudata as mu +import spatialdata as sd + +## VIASH START +par = { + "input": "./resources_test/xenium/xenium_tiny.h5mu", + "input_spatialdata": "./resources_test/xenium/xenium_tiny.zarr", + "output": "./resources_test/xenium/xenium_tiny_from_h5mu.zarr", + "modality": "rna", +} +meta = {"resources_dir": "src/utils"} +## VIASH END + +sys.path.append(meta["resources_dir"]) +from setup_logger import setup_logger + +logger = setup_logger() + +logger.info("Starting conversion from H5MU to SpatialData...") + +logger.info(f"Reading input H5MU file from {par['input']}...") +mdata = mu.read_h5mu(par["input"]) + +logger.info("Extracting modality from MuData object...") +mod = mdata.mod[par["modality"]] + +if par.get("input_spatialdata", None) is not None: + logger.info(f"Reading existing SpatialData from {par['input_spatialdata']}...") + + # Disable logger messages from spatialdata when reading + logger.setLevel(logging.WARNING) + sdata_existing = sd.read_zarr(par["input_spatialdata"]) + logger.setLevel(logging.INFO) + + logger.info("Checking modality matches existing SpatialData table...") + if not mod.n_obs == sdata_existing["table"].n_obs: + raise ValueError( + "The number of observations in the selected modality does not match the existing SpatialData table." + ) + if not mod.obs_names.equals(sdata_existing["table"].obs_names): + raise ValueError( + "The observation names in the selected modality do not match the existing SpatialData table." + ) + +logger.info("Creating SpatialData object...") +if par.get("input_spatialdata", None) is not None: + logger.info("Using existing SpatialData...") + sdata = sdata_existing + sdata["table"] = mod +else: + logger.info("Creating new SpatialData...") + sdata = sd.SpatialData(tables={"table": mod}) + +logger.info(f"Writing output SpatialData Zarr store to {par['output']}...") +sdata.write(par["output"], overwrite=True) + +logger.info("Done!") diff --git a/src/convert/from_h5mu_to_spatialdata/test.py b/src/convert/from_h5mu_to_spatialdata/test.py new file mode 100644 index 0000000..0d17d80 --- /dev/null +++ b/src/convert/from_h5mu_to_spatialdata/test.py @@ -0,0 +1,143 @@ +import os +import sys + +import mudata as mu +import pytest +import spatialdata as sd + + +def test_simple_execution(run_component, tmp_path): + input = meta["resources_dir"] + "/xenium_tiny.h5mu" + input_spatialdata = meta["resources_dir"] + "/xenium_tiny.zarr" + output = tmp_path / "output.zarr" + + run_component( + [ + "--input", + input, + "--input_spatialdata", + input_spatialdata, + "--output", + output, + ] + ) + assert os.path.exists(output), "output zarr was not created" + + mdata = mu.read_h5mu(input) + mod = mdata.mod["rna"] + + sdata = sd.read_zarr(output) + table = sdata["table"] + + # Check that the main table in the SpatialData object matches the selected modality in the MuData object + assert table.n_obs == mod.n_obs, ( + "The number of observations in the SpatialData table does not match the selected modality in the MuData object." + ) + assert table.obs_names.equals(mod.obs_names), ( + "The observation names in the SpatialData table do not match the selected modality in the MuData object." + ) + assert table.n_vars == mod.n_vars, ( + "The number of variables in the SpatialData table does not match the selected modality in the MuData object." + ) + assert table.var_names.equals(mod.var_names), ( + "The variable names in the SpatialData table do not match the selected modality in the MuData object." + ) + assert table.obs.keys().equals(mod.obs.keys()), ( + "The observation metadata columns in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert table.var.keys().equals(mod.var.keys()), ( + "The variable metadata columns in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.uns.keys()) == set(mod.uns.keys()), ( + "The unstructured metadata keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.obsm.keys()) == set(mod.obsm.keys()), ( + "The obsm keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.varm.keys()) == set(mod.varm.keys()), ( + "The varm keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + + sdata_existing = sd.read_zarr(input_spatialdata) + + # Check that remaining slots in the SpatialData object are filled from the existing SpatialData when provided + assert sdata.images.keys() == sdata_existing.images.keys(), ( + "The image keys in the output SpatialData do not match those in the existing SpatialData." + ) + assert sdata.labels.keys() == sdata_existing.labels.keys(), ( + "The label keys in the output SpatialData do not match those in the existing SpatialData." + ) + assert sdata.shapes.keys() == sdata_existing.shapes.keys(), ( + "The shape keys in the output SpatialData do not match those in the existing SpatialData." + ) + assert sdata.points.keys() == sdata_existing.points.keys(), ( + "The point keys in the output SpatialData do not match those in the existing SpatialData." + ) + + +def test_execution_without_input_spatialdata(run_component, tmp_path): + input = meta["resources_dir"] + "/xenium_tiny.h5mu" + output = tmp_path / "output_no_spatialdata.zarr" + + run_component( + [ + "--input", + input, + "--output", + output, + ] + ) + assert os.path.exists(output), "output zarr was not created" + + mdata = mu.read_h5mu(input) + mod = mdata.mod["rna"] + + sdata = sd.read_zarr(output) + table = sdata["table"] + + # Check that the main table in the SpatialData object matches the selected modality in the MuData object + assert table.n_obs == mod.n_obs, ( + "The number of observations in the SpatialData table does not match the selected modality in the MuData object." + ) + assert table.obs_names.equals(mod.obs_names), ( + "The observation names in the SpatialData table do not match the selected modality in the MuData object." + ) + assert table.n_vars == mod.n_vars, ( + "The number of variables in the SpatialData table does not match the selected modality in the MuData object." + ) + assert table.var_names.equals(mod.var_names), ( + "The variable names in the SpatialData table do not match the selected modality in the MuData object." + ) + assert table.obs.keys().equals(mod.obs.keys()), ( + "The observation metadata columns in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert table.var.keys().equals(mod.var.keys()), ( + "The variable metadata columns in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.uns.keys()) == set(mod.uns.keys()), ( + "The unstructured metadata keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.obsm.keys()) == set(mod.obsm.keys()), ( + "The obsm keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + assert set(table.varm.keys()) == set(mod.varm.keys()), ( + "The varm keys in the SpatialData table do not match those in the selected modality of the MuData object." + ) + + # When no existing SpatialData is provided, remaining slots should be empty + assert len(sdata.images) == 0, ( + "Expected no images when input_spatialdata is not provided." + ) + assert len(sdata.labels) == 0, ( + "Expected no labels when input_spatialdata is not provided." + ) + assert len(sdata.shapes) == 0, ( + "Expected no shapes when input_spatialdata is not provided." + ) + assert len(sdata.points) == 0, ( + "Expected no points when input_spatialdata is not provided." + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/src/convert/from_h5mu_to_spatialexperiment/config.vsh.yaml b/src/convert/from_h5mu_to_spatialexperiment/config.vsh.yaml index 42e9045..69b47ff 100644 --- a/src/convert/from_h5mu_to_spatialexperiment/config.vsh.yaml +++ b/src/convert/from_h5mu_to_spatialexperiment/config.vsh.yaml @@ -64,6 +64,7 @@ engines: - python3-pip - python3-dev - python-is-python3 + - cmake - type: r cran: [ reticulate, testthat ] - type: python diff --git a/src/convert/from_xenium_to_spatialexperiment/config.vsh.yaml b/src/convert/from_xenium_to_spatialexperiment/config.vsh.yaml index 12c49c4..348566b 100644 --- a/src/convert/from_xenium_to_spatialexperiment/config.vsh.yaml +++ b/src/convert/from_xenium_to_spatialexperiment/config.vsh.yaml @@ -68,8 +68,6 @@ engines: env: ["LIBARROW_MINIMAL=false"] - type: r script: 'install.packages("arrow", type = "source")' - - type: r - script: 'Sys.setenv(LIBARROW_MINIMAL = "false"); install.packages("arrow", type = "source")' - type: r bioc: [ SpatialExperimentIO ] test_setup: diff --git a/src/dataflow/split_h5mu/config.vsh.yaml b/src/dataflow/split_h5mu/config.vsh.yaml deleted file mode 100644 index 0bf8dab..0000000 --- a/src/dataflow/split_h5mu/config.vsh.yaml +++ /dev/null @@ -1,73 +0,0 @@ -name: split_h5mu -namespace: "dataflow" -scope: "public" -description: | - Split the samples of a single modality from a .h5mu (multimodal) sample into seperate .h5mu files based on the values of an .obs column of this modality. -authors: - - __merge__: /src/authors/dorien_roosen.yaml - roles: [ author, maintainer ] -argument_groups: -- name: Input & specifications - arguments: - - name: "--input" - type: file - description: Path to a single .h5mu file. - required: true - - name: "--modality" - description: | - Which modality from the input MuData file to process. - type: string - default: "rna" - required: false - - name: "--obs_feature" - type: string - required: true - description: The .obs column to split the mudata on. - example: "celltype" - - name: "--drop_obs_nan" - type: boolean_true - description: Whether to drop all .obs columns that contain only nan values after splitting. - - name: "--ensure_unique_filenames" - type: boolean_true - description: Append number suffixes to ensure unique filenames after sanitizing obs feature values. - -- name: Outputs - arguments: - - name: "--output" - type: file - required: true - direction: output - example: "/path/to/output" - description: Output directory containing multiple h5mu files. - - name: "--output_files" - type: file - required: true - direction: output - example: sample_files.csv - description: A csv containing the base filename and obs feature by which it was split. - __merge__: [., /src/base/h5_compression_argument.yaml] - - -resources: - - type: python_script - path: script.py - - path: /src/utils/setup_logger.py -test_resources: - - type: python_script - path: test.py - -engines: - - type: docker - image: python:3.12-slim - setup: - - type: apt - packages: - - procps - - type: python - __merge__: /src/base/requirements/anndata_mudata.yaml - __merge__: [ /src/base/requirements/python_test_setup.yaml, .] -runners: - - type: executable - - type: nextflow - directives: - label: [ lowcpu, highmem, highdisk] diff --git a/src/dataflow/split_h5mu/script.py b/src/dataflow/split_h5mu/script.py deleted file mode 100644 index a166fc2..0000000 --- a/src/dataflow/split_h5mu/script.py +++ /dev/null @@ -1,119 +0,0 @@ -import sys -import mudata as mu -import pandas as pd -import re -import gc -from pathlib import Path -from collections import defaultdict - -### VIASH START -par = { - "input": "cart_atomx_process_samples.h5mu", - "modality": "rna", - "obs_feature": "sample_id", - "output": "reference_download/sample_split", - "drop_obs_nan": "true", - "output_compression": None, - "output_files": "reference_download/sample_files.csv", - "ensure_unique_filenames": True, -} -meta = { - "resources_dir": "src/utils", -} -# import anndata as ad - -# df = pd.DataFrame( -# [[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var1", "var2", "var3"] -# ) -# var3 = pd.DataFrame(["d", "e", "g"], index=df.columns, columns=["Feat"]) -# obs3 = pd.DataFrame(["C C", "C_C"], index=df.index, columns=["Obs"]) -# ad3 = ad.AnnData(df, obs=obs3, var=var3) -# mdata = mu.MuData({"rna": ad3}) -# mdata.write_h5mu("test_san.h5mu") -# par["input"] = "test_san.h5mu" -# par["obs_feature"] = "Obs" -### VIASH END - -sys.path.append(meta["resources_dir"]) -from setup_logger import setup_logger - -logger = setup_logger() - - -def main(): - logger.info(f"Reading {par['input']}") - input_file = Path(par["input"].strip()) - - mdata = mu.read_h5mu(input_file) - adata = mdata.mod[par["modality"]] - - logger.info(f"Reading unique features from {par['obs_feature']}") - obs_features = adata.obs[par["obs_feature"]].unique().tolist() - - # sanitize --obs_feature values - obs_features_s = [re.sub(r"[-\s]", "_", str(s).strip()) for s in obs_features] - obs_features_s = [re.sub(r"[^A-Za-z0-9_]", "", s) for s in obs_features_s] - - # ensure that names are unique, if not raise or append number as suffix - if not len(obs_features_s) == len(set(obs_features_s)): - if not par["ensure_unique_filenames"]: - raise ValueError( - f"File names are not unique after sanitizing the --obs_feature {par['obs_feature']} values" - ) - - logger.info("Ensuring unique names for par['obs_feature']") - counts = defaultdict(lambda: -1) - for i, feature in enumerate(obs_features_s): - counts[feature] += 1 - if (curr_counts := counts[feature]) > 0: - obs_features_s[i] += f"_{curr_counts}" - - # generate output dir - output_dir = Path(par["output"]) - if not output_dir.is_dir(): - output_dir.mkdir(parents=True) - - # split modality of mdata file base on obs_feature - logger.info(f"Splitting file based on {par['obs_feature']} values {obs_features}") - obs_files = [] - - for obs_name, file_name in zip(obs_features, obs_features_s): - logger.info( - f"Filtering modality '{par['modality']}' observations by .obs['{par['obs_feature']}'] == {obs_name}" - ) - mdata_obs = mdata.copy() - adata_full = mdata_obs.mod[par["modality"]] - - # split the samples - mask = adata_full.obs[par["obs_feature"]] == obs_name - adata_obs = adata_full[mask].copy() - - # Dropping columns that only have nan values after splitting - if par["drop_obs_nan"]: - logger.info("Dropping all .obs columns with NaN values") - adata_obs.obs = adata_obs.obs.dropna(axis=1, how="all") - - mdata_obs.mod[par["modality"]] = adata_obs - - mdata_obs_name = f"{input_file.stem}_{file_name}.h5mu" - out_path = output_dir / mdata_obs_name - - # replace mdata file with modality adata contianing split samples - logger.info( - f"Writing h5mu filtered for {par['obs_feature']} {obs_name} to file {out_path}" - ) - - mdata_obs.write_h5mu(out_path, compression=par["output_compression"]) - - # avoid keeping files in memory - obs_files.append(mdata_obs_name) - del mdata_obs, adata_obs - gc.collect() - - logger.info(f"Writing output_files CSV file to {par['output_files']}") - df = pd.DataFrame({"name": obs_features_s, "filename": obs_files}) - df.to_csv(par["output_files"], index=False) - - -if __name__ == "__main__": - main() diff --git a/src/dataflow/split_h5mu/test.py b/src/dataflow/split_h5mu/test.py deleted file mode 100644 index 6fdab5c..0000000 --- a/src/dataflow/split_h5mu/test.py +++ /dev/null @@ -1,288 +0,0 @@ -import sys -from textwrap import dedent -import pytest -import mudata as mu -import anndata as ad -import numpy as np -import pandas as pd -import subprocess -import re - - -@pytest.fixture -def input_modality_1(): - df = pd.DataFrame( - [[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var1", "var2", "var3"] - ) - obs = pd.DataFrame({"Obs": ["A", "B"], "Obs_nan": [np.nan, np.nan]}, index=df.index) - var = pd.DataFrame([["a"], ["b"], ["c"]], index=df.columns, columns=["Feat"]) - ad1 = ad.AnnData(df, obs=obs, var=var) - return ad1 - - -@pytest.fixture -def input_modality_2(): - df = pd.DataFrame( - [[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var1", "var2", "var3"] - ) - var2 = pd.DataFrame(["d", "e", "g"], index=df.columns, columns=["Feat"]) - obs2 = pd.DataFrame(["C", "D"], index=df.index, columns=["Obs"]) - ad2 = ad.AnnData(df, obs=obs2, var=var2) - return ad2 - - -@pytest.fixture -def input_modality_3(): - df = pd.DataFrame( - [[1, 2, 3], [4, 5, 6]], index=["obs1", "obs2"], columns=["var1", "var2", "var3"] - ) - var3 = pd.DataFrame(["d", "e", "g"], index=df.columns, columns=["Feat"]) - obs3 = pd.DataFrame(["C C", "C_C"], index=df.index, columns=["Obs"]) - ad3 = ad.AnnData(df, obs=obs3, var=var3) - return ad3 - - -@pytest.fixture -def input_h5mu(input_modality_1, input_modality_2): - tmp_mudata = mu.MuData({"mod1": input_modality_1, "mod2": input_modality_2}) - return tmp_mudata - - -@pytest.fixture -def input_h5mu_path(write_mudata_to_file, input_h5mu): - return write_mudata_to_file(input_h5mu) - - -@pytest.fixture -def input_h5mu_non_unique_filenames(input_modality_3): - tmp_mudata = mu.MuData({"mod3": input_modality_3}) - return tmp_mudata - - -@pytest.fixture -def input_h5mu_path_non_unique_filenames( - write_mudata_to_file, input_h5mu_non_unique_filenames -): - return write_mudata_to_file(input_h5mu_non_unique_filenames) - - -def test_sample_split(run_component, random_path, input_h5mu, input_h5mu_path): - output_dir = random_path() - output_files = random_path(extension="csv") - args = [ - "--input", - input_h5mu_path, - "--output", - str(output_dir), - "--modality", - "mod1", - "--obs_feature", - "Obs", - "--output_files", - str(output_files), - ] - - run_component(args) - assert output_files.is_file() - assert output_dir.is_dir() - - # check output dir and file names - dir_content = [ - h5mu_file - for h5mu_file in output_dir.iterdir() - if h5mu_file.suffix == ".h5mu" and h5mu_file != input_h5mu_path - ] - s1_file = output_dir / f"{input_h5mu_path.stem}_A.h5mu" - s2_file = output_dir / f"{input_h5mu_path.stem}_B.h5mu" - assert set(dir_content) == set([s1_file, s2_file]) - - # check that number of modalities, variables and observations - s1 = mu.read_h5mu(s1_file) - s2 = mu.read_h5mu(s2_file) - assert s1.n_mod == 2 - assert s2.n_mod == 2 - - assert s1.n_obs == input_h5mu.n_obs, ( - "number of observations of split file does not match input file" - ) - assert s2.n_obs == input_h5mu.n_obs, ( - "number of observations of split file does not match input file" - ) - - assert s1.mod["mod1"].n_obs == 1, ( - "number of observations of split file s1 modality mod1 should equal 1" - ) - assert s1.mod["mod2"].n_obs == input_h5mu.n_obs, ( - "number of observations of split file s1 modality mod2 should equal input file" - ) - - assert len(s1.mod["mod1"].obs.keys()) == 2, ( - "number of observation keys split file s1 modality mod1 should equal 2" - ) - assert len(s1.mod["mod2"].obs.keys()) == 1, ( - "number of observation keys split file s1 modality mod2 should equal 1" - ) - - assert s2.mod["mod1"].n_obs == 1, ( - "number of observations of split file s2 modality mod1 should equal 1" - ) - assert s2.mod["mod2"].n_obs == input_h5mu.n_obs, ( - "number of observations of split file s2 modality mod2 should equal input file" - ) - - assert s1.n_vars == input_h5mu.n_vars, ( - "number of variables of split file s1 should equal input file" - ) - assert s2.n_vars == input_h5mu.n_vars, ( - "number of variables of split file s1 should equal input file" - ) - - assert s1.mod["mod1"].n_vars == input_h5mu.mod["mod1"].n_vars, ( - "number of variables of split file s1 modalitty mod1 should equal input file" - ) - assert s1.mod["mod2"].n_vars == input_h5mu.mod["mod1"].n_vars, ( - "number of variables of split file s1 modalitty mod2 should equal input file" - ) - - assert s2.mod["mod1"].n_vars == input_h5mu.mod["mod1"].n_vars, ( - "number of variables of split file s2 modalitty mod1 should equal input file" - ) - assert s2.mod["mod2"].n_vars == input_h5mu.mod["mod1"].n_vars, ( - "number of variables of split file s2 modalitty mod2 should equal input file" - ) - - # check correct sample splitting - assert np.all(s1.mod["mod1"].obs["Obs"] == "A"), ( - "observation of .obs Obs in s1 should equal A" - ) - assert np.all(s2.mod["mod1"].obs["Obs"] == "B"), ( - "observation of .obs Obs in s2 should equal B" - ) - - # Check contents of csv file - expected_csv_output = dedent( - f"""\ - name,filename - A,{s1_file.name} - B,{s2_file.name} - """ - ) - with open(output_files, "r") as open_csv_file: - result = open_csv_file.read() - assert result == expected_csv_output - - -def test_sample_split_dropna(run_component, random_path, input_h5mu, input_h5mu_path): - output_dir = random_path() - output_files = random_path(extension="csv") - args = [ - "--input", - input_h5mu_path, - "--output", - str(output_dir), - "--modality", - "mod1", - "--obs_feature", - "Obs", - "--drop_obs_nan", - "true", - "--output_files", - str(output_files), - ] - - run_component(args) - - # check output dir and file names - s1_file = output_dir / f"{input_h5mu_path.stem}_A.h5mu" - s2_file = output_dir / f"{input_h5mu_path.stem}_B.h5mu" - - # check that .obs columns with only nan values are dropped correctly - s1 = mu.read_h5mu(s1_file) - s2 = mu.read_h5mu(s2_file) - - assert s1.n_obs == input_h5mu.n_obs, ( - "number of observations of split file does not match input file" - ) - assert s2.n_obs == input_h5mu.n_obs, ( - "number of observations of split file does not match input file" - ) - - assert s1.mod["mod1"].n_obs == 1, ( - "number of observations of split file s1 modality mod1 should equal 1" - ) - assert s1.mod["mod2"].n_obs == input_h5mu.n_obs, ( - "number of observations of split file s1 modality mod2 should equal input file" - ) - - assert len(s1.mod["mod1"].obs.keys()) == 1, ( - "number of observation keys split file s1 modality mod1 should equal 1" - ) - assert len(s1.mod["mod2"].obs.keys()) == 1, ( - "number of observation keys split file s1 modality mod2 should equal 1" - ) - - -def test_sanitizing(run_component, random_path, input_h5mu_path_non_unique_filenames): - output_dir = random_path() - output_files = random_path(extension="csv") - - args = [ - "--input", - input_h5mu_path_non_unique_filenames, - "--output", - str(output_dir), - "--modality", - "mod3", - "--obs_feature", - "Obs", - "--drop_obs_nan", - "true", - "--output_files", - str(output_files), - ] - - with pytest.raises(subprocess.CalledProcessError) as err: - run_component(args) - assert re.search( - r"ValueError: File names are not unique after sanitizing the --obs_feature Obs values", - err.value.stdout.decode("utf-8"), - ) - - args_san = [ - "--input", - input_h5mu_path_non_unique_filenames, - "--output", - str(output_dir), - "--modality", - "mod3", - "--obs_feature", - "Obs", - "--drop_obs_nan", - "true", - "--output_files", - str(output_files), - "--ensure_unique_filenames", - "true", - ] - - run_component(args_san) - - # check output dir and file names - dir_content = [ - h5mu_file - for h5mu_file in output_dir.iterdir() - if h5mu_file.suffix == ".h5mu" - and h5mu_file != input_h5mu_path_non_unique_filenames - ] - s1_file = output_dir / f"{input_h5mu_path_non_unique_filenames.stem}_C_C.h5mu" - s2_file = output_dir / f"{input_h5mu_path_non_unique_filenames.stem}_C_C_1.h5mu" - - assert s1_file.is_file(), f"{s1_file} does not exist" - assert s2_file.is_file(), f"{s2_file} does not exist" - assert set(dir_content) == set([s1_file, s2_file]), ( - "Output files do not match file names in csv" - ) - - -if __name__ == "__main__": - sys.exit(pytest.main([__file__])) diff --git a/src/feature_annotation/spatial_autocorr/config.vsh.yaml b/src/feature_annotation/spatial_autocorr/config.vsh.yaml new file mode 100644 index 0000000..3ae7c03 --- /dev/null +++ b/src/feature_annotation/spatial_autocorr/config.vsh.yaml @@ -0,0 +1,111 @@ +name: spatial_autocorr +namespace: feature_annotation +label: Spatial Autocorrelation +summary: Calculate spatial autocorrelation for genes using Moran's I or Geary's C. +description: | + Calculate spatial autocorrelation for genes using Moran's I or Geary's C. + This allows identifying spatially variable genes. + +argument_groups: + - name: Inputs + arguments: + - name: --input + type: file + required: true + description: Input h5mu file. + - name: --modality + type: string + default: rna + description: Modality to use. + - name: --layer + type: string + description: | + Layer in the AnnData object to use. + - If `attr` is 'X', this is the key in `.layers` to use. If not provided, `.X` is used. + - If `attr` is 'obsm', this is the key in `.obsm` to use. + - name: --input_genes + type: string + multiple: true + multiple_sep: "," + description: | + The features to calculate autocorrelation for. Meaning depends on `--attr`: + - If `attr` is 'X' (default), this must be a list of gene names from `.var_names`. + If not provided, it defaults to using genes in `.var['highly_variable']` (if present), or all genes. + This behavior can be overridden by setting `--use_all_genes` to `true`. + Note: You cannot pass a column name from `.var` here. + - If `attr` is 'obs', this must be a list of column names from `.obs`. + - If `attr` is 'obsm', this must be indices in `.obsm[layer]` (as strings). + - name: --obsp_neighborhood_graph + type: string + default: spatial_connectivities + description: Key in .obsp where spatial connectivities are stored. + - name: --use_all_genes + type: boolean + default: false + description: | + Whether to use all genes even if highly variable genes are present in .var. + If set to true, all genes will be used. + + - name: Parameters + arguments: + - name: --mode + type: string + default: moran + choices: [moran, geary] + description: Mode of spatial autocorrelation. + - name: --attr + type: string + default: X + choices: [X, obs, obsm] + description: | + The attribute of the AnnData object to use for calculation. + - 'X': Use gene expression data (default). + - 'obs': Use cell metadata. + - 'obsm': Use multidimensional embeddings. + - name: --n_perms + type: integer + default: 100 + description: Number of permutations for p-value calculation. + - name: --use_raw + type: boolean + default: false + description: Whether to use .raw attribute of AnnData. + + - name: Outputs + arguments: + - name: --output + type: file + direction: output + required: true + description: Output h5mu file with results in .uns. + +resources: + - type: python_script + path: script.py + +test_resources: + - type: python_script + path: test.py + - path: /resources_test/xenium/xenium_tiny.qc.neighbors.h5mu + +engines: + - type: docker + image: python:3.13-slim + setup: + - type: apt + packages: + - procps + - type: python + __merge__: [/src/base/requirements/anndata_mudata.yaml, /src/base/requirements/scanpy.yaml, /src/base/requirements/squidpy.yaml] + test_setup: + - type: python + packages: + - pytest + - viashpy + - type: native + +runners: + - type: executable + - type: nextflow + directives: + label: [midcpu, midmem] diff --git a/src/feature_annotation/spatial_autocorr/script.py b/src/feature_annotation/spatial_autocorr/script.py new file mode 100644 index 0000000..09bf38b --- /dev/null +++ b/src/feature_annotation/spatial_autocorr/script.py @@ -0,0 +1,80 @@ +import mudata as mu +import squidpy as sq + +## VIASH START +par = { + "input": "resources_test/xenium/xenium_tiny_qc_graph.h5mu", + "output": "output.h5mu", + "modality": "rna", + "mode": "moran", + "input_genes": None, + "attr": "X", + "n_perms": 100, + "layer": None, + "use_raw": False, + "obsp_neighborhood_graph": "spatial_connectivities", + "use_all_genes": False, +} +## VIASH END + + +def main(): + print("Reading input MuData...", flush=True) + mdata = mu.read_h5mu(par["input"]) + adata = mdata.mod[par["modality"]] + + # Check for connectivity key + if par["obsp_neighborhood_graph"] not in adata.obsp: + raise ValueError( + f"Connectivity key '{par['obsp_neighborhood_graph']}' not found in .obsp of modality '{par['modality']}'." + ) + + genes = par["input_genes"] + if genes and len(genes) == 0: + genes = None + + if genes is None and par["use_all_genes"] and par["attr"] == "X": + genes = list(adata.var_names) + + # Handle layer + layer = par["layer"] + if layer == "None": + layer = None + + print(f"Calculating spatial autocorrelation ({par['mode']})...", flush=True) + + # Run Squidpy spatial_autocorr + # Note: sq.gr.spatial_autocorr modifies adata in-place, adding results to .uns + sq.gr.spatial_autocorr( + adata, + connectivity_key=par["obsp_neighborhood_graph"], + genes=genes, + mode=par["mode"], + attr=par["attr"], + n_perms=par["n_perms"], + layer=layer, + use_raw=par["use_raw"], + ) + + result_key = f"{par['mode']}I" if par["mode"] == "moran" else f"{par['mode']}C" + + if result_key in adata.uns: + # Log top spatially variable genes + df = adata.uns[result_key] + if not df.empty: + sort_col = "I" if par["mode"] == "moran" else "C" + print("Top spatially variable genes:", flush=True) + print(df.sort_values(by=sort_col, ascending=False).head(), flush=True) + else: + print( + f"Warning: Expected key '{result_key}' not found in .uns after calculation.", + flush=True, + ) + + print("Writing output...", flush=True) + mdata.write_h5mu(par["output"]) + print("Done!", flush=True) + + +if __name__ == "__main__": + main() diff --git a/src/feature_annotation/spatial_autocorr/test.py b/src/feature_annotation/spatial_autocorr/test.py new file mode 100644 index 0000000..ca13fc5 --- /dev/null +++ b/src/feature_annotation/spatial_autocorr/test.py @@ -0,0 +1,220 @@ +import mudata as mu +import sys +import pytest +import pandas as pd + +## VIASH START +meta = { + "resources_dir": "resources_test/xenium", + "executable": "./calculate_spatial_autocorrelation", +} +## VIASH END + + +def test_calculate_spatial_autocorrelation_moran(run_component, tmp_path): + input_path = meta["resources_dir"] + "/xenium_tiny.qc.neighbors.h5mu" + output_path = tmp_path / "output_moran.h5mu" + + print(f"Running component with Moran's I on {input_path}") + + run_component( + [ + "--input", + str(input_path), + "--output", + str(output_path), + "--modality", + "rna", + "--obsp_neighborhood_graph", + "spatial_connectivities", + "--mode", + "moran", + "--n_perms", + "10", # Reduce permutations for speed + ] + ) + + assert output_path.exists(), "Output file not created" + + mdata = mu.read_h5mu(output_path) + adata = mdata.mod["rna"] + + assert "moranI" in adata.uns, "moranI key missing from .uns" + df = adata.uns["moranI"] + assert isinstance(df, pd.DataFrame), "moranI should be a DataFrame" + assert not df.empty, "moranI DataFrame is empty" + + # Check essential columns + expected_cols = ["I", "pval_norm", "var_norm"] + for col in expected_cols: + assert col in df.columns, f"Missing column {col} in moranI dataframe" + + # Check values range + assert df["I"].max() <= 1.0, "Moran's I > 1 (theoretical max)" + assert df["I"].min() >= -1.0, "Moran's I < -1 (theoretical min)" + + +def test_calculate_spatial_autocorrelation_geary(run_component, tmp_path): + input_path = meta["resources_dir"] + "/xenium_tiny.qc.neighbors.h5mu" + output_path = tmp_path / "output_geary.h5mu" + + mdata = mu.read_h5mu(input_path) + genes = list(mdata.mod["rna"].var_names[:5]) # Test with first 5 genes + genes_str = ",".join(genes) + + cmd = [ + "--input", + str(input_path), + "--output", + str(output_path), + "--modality", + "rna", + "--obsp_neighborhood_graph", + "spatial_connectivities", + "--mode", + "geary", + "--n_perms", + "10", + "--input_genes", + genes_str, + ] + + print( + f"Running component with Geary's C on {input_path} for subset of genes: {genes_str}" + ) + run_component(cmd) + + assert output_path.exists() + + mdata = mu.read_h5mu(output_path) + adata = mdata.mod["rna"] + + assert "gearyC" in adata.uns + df = adata.uns["gearyC"] + + assert len(df) == 5, f"Expected results for 5 genes, got {len(df)}" + assert all(g in df.index for g in genes), ( + "Not all requested genes are in output index" + ) + + assert "C" in df.columns + + +def test_calculate_spatial_autocorrelation_obs(run_component, tmp_path): + input_path = meta["resources_dir"] + "/xenium_tiny.qc.neighbors.h5mu" + output_path = tmp_path / "output_obs.h5mu" + + features = "total_counts,cell_area,nucleus_area" + + print( + f"Running component with Moran's I on {input_path} for obs features: {features}" + ) + + run_component( + [ + "--input", + str(input_path), + "--output", + str(output_path), + "--modality", + "rna", + "--obsp_neighborhood_graph", + "spatial_connectivities", + "--mode", + "moran", + "--attr", + "obs", + "--input_genes", + features, + "--n_perms", + "10", + ] + ) + + assert output_path.exists() + + mdata = mu.read_h5mu(output_path) + adata = mdata.mod["rna"] + + assert "moranI" in adata.uns + df = adata.uns["moranI"] + + assert len(df) == 3, f"Expected results for 3 features, got {len(df)}" + # When attr='obs', the index contains the column names + for feature in features.split(","): + assert feature in df.index, f"Feature {feature} missing from index: {df.index}" + + +def test_calculate_spatial_autocorrelation_use_all_genes(run_component, tmp_path): + input_path = meta["resources_dir"] + "/xenium_tiny.qc.neighbors.h5mu" + temp_input_path = tmp_path / "xenium_tiny_hvg.h5mu" + output_path_hvg = tmp_path / "output_hvg.h5mu" + output_path_all = tmp_path / "output_all.h5mu" + + # Create dataset with highly variable genes + mdata = mu.read_h5mu(input_path) + adata = mdata.mod["rna"] + + # Mark first 5 genes as highly variable + adata.var["highly_variable"] = False + genes = list(adata.var_names) + for g in genes[:5]: + adata.var.loc[g, "highly_variable"] = True + + mdata.write_h5mu(temp_input_path) + + # Test 1: Default behavior (should use only HVG) + print("Running component with default settings (expecting HVG usage)") + run_component( + [ + "--input", + str(temp_input_path), + "--output", + str(output_path_hvg), + "--modality", + "rna", + "--obsp_neighborhood_graph", + "spatial_connectivities", + "--mode", + "moran", + "--n_perms", + "10", + ] + ) + + assert output_path_hvg.exists() + mdata_hvg = mu.read_h5mu(output_path_hvg) + df_hvg = mdata_hvg.mod["rna"].uns["moranI"] + assert len(df_hvg) == 5, f"Expected 5 genes (HVG), got {len(df_hvg)}" + + # Test 2: Override to use all genes + print("Running component with --use_all_genes (expecting all genes)") + run_component( + [ + "--input", + str(temp_input_path), + "--output", + str(output_path_all), + "--modality", + "rna", + "--obsp_neighborhood_graph", + "spatial_connectivities", + "--mode", + "moran", + "--n_perms", + "10", + "--use_all_genes", + "true", + ] + ) + + assert output_path_all.exists() + mdata_all = mu.read_h5mu(output_path_all) + df_all = mdata_all.mod["rna"].uns["moranI"] + assert len(df_all) == len(genes), ( + f"Expected all {len(genes)} genes, got {len(df_all)}" + ) + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml b/src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml new file mode 100644 index 0000000..37d0abe --- /dev/null +++ b/src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml @@ -0,0 +1,109 @@ +name: xenium_spatial_statistics +namespace: feature_annotation +label: Xenium Spatial Statistics +description: | + Calculate various spatial statistics from Xenium data including cell morphology + ratios, position-based features, local density metrics, and global spatial patterns. + This component expects a MuData object with a pre-computed spatial neighborhood graph + in `.obsp`. + +argument_groups: + - name: Inputs + arguments: + - name: --input + type: file + required: true + example: input.h5mu + description: | + A MuData file containing spatial transcriptomics data and a + pre-computed spatial neighborhood graph in `.obsp`. + + - name: --modality + type: string + required: true + default: "rna" + description: | + The name of the modality to use from the MuData object. + This specifies which AnnData object contains the spatial data. + + - name: --obsm_spatial_coordinates + type: string + default: "spatial" + description: | + The key in `.obsm` where spatial coordinates are stored. + Expected shape is (n_cells, 2) for 2D coordinates. + + - name: Outputs + arguments: + - name: --output + type: file + direction: output + default: output.h5mu + description: | + The output MuData file with calculated spatial statistics. + Per-cell metrics are stored in `.obs` and global statistics in `.uns`. + + - name: --output_prefix + type: string + default: "spatial_" + description: | + Prefix to add to all generated column names in `.obs`. + + - name: Parameters + arguments: + - name: --density_bandwidth + type: double + default: 50.0 + description: | + Bandwidth parameter for Gaussian kernel density estimation in spatial units + (typically microns). Larger values create smoother density estimates. + + - name: "--calculate_ripley_l" + type: boolean + default: false + description: | + Whether to calculate Ripley's L statistic. + Warning: This is an O(N^2) operation and can be very slow for large datasets (>5000 cells). + + - name: "--n_subsample_ripley" + type: integer + default: -1 + description: | + Number of cells to subsample for Ripley's L calculation. + If -1, use all cells. Recommended to keep this below 5000 for performance. + +resources: + - type: python_script + path: script.py + +test_resources: + - type: python_script + path: test.py + - path: /resources_test/xenium/xenium_tiny.qc.neighbors.h5mu + +engines: + - type: docker + image: python:3.13-slim + setup: + - type: apt + packages: + - procps + - type: python + __merge__: + - /src/base/requirements/anndata_mudata.yaml + - /src/base/requirements/scanpy.yaml + - /src/base/requirements/squidpy.yaml + packages: + - scikit-learn + test_setup: + - type: python + packages: + - pytest + - viashpy + - type: native + +runners: + - type: executable + - type: nextflow + directives: + label: [singlecpu, midmem, lowdisk] diff --git a/src/feature_annotation/xenium_spatial_statistics/script.py b/src/feature_annotation/xenium_spatial_statistics/script.py new file mode 100644 index 0000000..bf768ae --- /dev/null +++ b/src/feature_annotation/xenium_spatial_statistics/script.py @@ -0,0 +1,315 @@ +import sys +import warnings + +import mudata as mu +import numpy as np +import squidpy as sq +from scipy.spatial import ConvexHull, Voronoi, distance_matrix +from sklearn.neighbors import KernelDensity + +## VIASH START +par = { + "input": "resources_test/xenium/xenium_tiny_qc.h5mu", + "output": "output.h5mu", + "modality": "rna", + "obsm_spatial_coordinates": "spatial", + "density_bandwidth": 50.0, + "calculate_ripley_l": False, + "n_subsample_ripley": -1, + "output_prefix": "spatial_", +} +## VIASH END + + +def calculate_morphology_metrics(adata, prefix): + """Calculate cell morphology ratios and features.""" + print(" Calculating morphology metrics...", flush=True) + + if "cell_area" not in adata.obs or "nucleus_area" not in adata.obs: + warnings.warn( + "cell_area and/or nucleus_area not found in .obs. " + "Skipping morphology metrics." + ) + return + + # Nucleus to cell area ratio - can be removed once https://github.com/openpipelines-bio/openpipeline_qc/issues/18 is fixed. + adata.obs[f"{prefix}nucleus_cell_ratio"] = ( + adata.obs["nucleus_area"] / adata.obs["cell_area"] + ) + + # Cell area percentile (relative size) + adata.obs[f"{prefix}cell_area_percentile"] = ( + adata.obs["cell_area"].rank(pct=True) * 100 + ) + + print( + f" Added: {prefix}nucleus_cell_ratio, {prefix}cell_area_percentile", + flush=True, + ) + + +def calculate_position_features(adata, spatial_coords, prefix): + """Calculate position-based features.""" + print(" Calculating position-based features...", flush=True) + + # Tissue centroid + centroid = spatial_coords.mean(axis=0) + + # Distance to centroid + distances_to_centroid = np.linalg.norm(spatial_coords - centroid, axis=1) + adata.obs[f"{prefix}distance_to_centroid"] = distances_to_centroid + + # Normalized coordinates (0-1 scale) + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + coord_range = max_coords - min_coords + + normalized_coords = (spatial_coords - min_coords) / coord_range + adata.obs[f"{prefix}norm_x"] = normalized_coords[:, 0] + adata.obs[f"{prefix}norm_y"] = normalized_coords[:, 1] + + # Distance to convex hull boundary + try: + hull = ConvexHull(spatial_coords) + hull_points = spatial_coords[hull.vertices] + + # For each point, find distance to nearest hull vertex (approximation) + distances_to_boundary = np.min( + distance_matrix(spatial_coords, hull_points), axis=1 + ) + adata.obs[f"{prefix}distance_to_boundary"] = distances_to_boundary + print( + " Added: distance_to_centroid, norm_x, norm_y, distance_to_boundary", + flush=True, + ) + except Exception as e: + warnings.warn(f"Could not calculate convex hull: {e}") + print(" Added: distance_to_centroid, norm_x, norm_y", flush=True) + + +def calculate_density_metrics(adata, spatial_coords, bandwidth, prefix): + """Calculate local density metrics.""" + print(" Calculating density metrics...", flush=True) + + # Kernel density estimation + kde = KernelDensity(bandwidth=bandwidth, kernel="gaussian") + kde.fit(spatial_coords) + log_density = kde.score_samples(spatial_coords) + adata.obs[f"{prefix}kernel_density"] = np.exp(log_density) + + # Calculate degree centrality (as a proxy for local density / number of neighbors) + # This assumes spatial_connectivities is already present in .obsp (from build_spatial_graph) + if "spatial_connectivities" in adata.obsp: + spatial_connectivities = adata.obsp["spatial_connectivities"] + degrees = spatial_connectivities.sum(axis=1) + # If sparse matrix, it returns matrix object, need to convert to array + if hasattr(degrees, "A1"): + degrees = degrees.A1 + adata.obs[f"{prefix}graph_degree"] = degrees + + print(" Added: kernel_density, graph_degree", flush=True) + else: + print( + " Warning: 'spatial_connectivities' not found in .obsp. Skipping graph_degree.", + flush=True, + ) + print(" Added: kernel_density", flush=True) + + +def calculate_voronoi_metrics(adata, spatial_coords, prefix): + """Calculate Voronoi tessellation statistics.""" + print(" Calculating Voronoi tessellation...", flush=True) + + try: + vor = Voronoi(spatial_coords) + + polygon_areas = [] + neighbor_counts = [] + + for point_idx in range(len(spatial_coords)): + region_idx = vor.point_region[point_idx] + region = vor.regions[region_idx] + + # Skip infinite regions + if -1 in region or len(region) == 0: + polygon_areas.append(np.nan) + neighbor_counts.append(np.nan) + continue + + # Calculate polygon area using shoelace formula + vertices = vor.vertices[region] + x = vertices[:, 0] + y = vertices[:, 1] + area = 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + polygon_areas.append(area) + + # Count neighbors (shared vertices) + neighbor_counts.append(len(region)) + + adata.obs[f"{prefix}voronoi_area"] = polygon_areas + adata.obs[f"{prefix}voronoi_neighbors"] = neighbor_counts + + print(" Added: voronoi_area, voronoi_neighbors", flush=True) + except Exception as e: + warnings.warn(f"Could not calculate Voronoi tessellation: {e}") + + +def calculate_global_statistics(adata, spatial_coords, par): + """Calculate global spatial pattern statistics.""" + print(" Calculating global spatial statistics...", flush=True) + + stats = {} + + # Global density + try: + hull = ConvexHull(spatial_coords) + area = hull.volume # In 2D, volume of convex hull is the area + stats["area_calculation_method"] = "convex_hull" + stats["cell_density"] = len(spatial_coords) / area + stats["total_area"] = area + stats["n_cells"] = len(spatial_coords) + except Exception as e: + print(f" Error: Could not calculate convex hull area ({e})", flush=True) + + # Ripley's L (using squidpy) + if par["calculate_ripley_l"]: + print(" Computing Ripley's L function (Squidpy)...", flush=True) + + n_subsample = par["n_subsample_ripley"] + + try: + # Create a working AnnData for Ripley's + if n_subsample > 0 and len(adata) > n_subsample: + print(f" Subsampling to {n_subsample} random cells...", flush=True) + + # Use numpy to generate random indices + indices = np.random.choice(len(adata), n_subsample, replace=False) + adata_ripley = adata[indices].copy() + stats["ripley_l_subsampled"] = True + else: + adata_ripley = adata.copy() + stats["ripley_l_subsampled"] = False + + # Squidpy requires a cluster key. We create a dummy one for "global" context. + adata_ripley.obs["_temp_global"] = "all" + adata_ripley.obs["_temp_global"] = adata_ripley.obs["_temp_global"].astype( + "category" + ) + + # Calculate Ripley's L statistic + # This stores the result in adata.uns['all_L'] + sq.gr.ripley( + adata_ripley, + cluster_key="_temp_global", + mode="L", + spatial_key=par["obsm_spatial_coordinates"], + n_simulations=50, + ) + + # Extract basic stats from the results + if "all_L" in adata_ripley.uns and "L_stat" in adata_ripley.uns["all_L"]: + l_stat_df = adata_ripley.uns["all_L"]["L_stat"] + stats["ripley_l_max"] = float(l_stat_df.max().max()) + stats["ripley_l_mean"] = float(l_stat_df.mean().mean()) + + # Clean up + if "all_L" in adata_ripley.uns: + del adata_ripley.uns["all_L"] + + except Exception as e: + print(f" Error calculating Ripley's L: {e}", flush=True) + import traceback + + traceback.print_exc() + else: + print(" Skipping Ripley's L (disabled in config)...", flush=True) + + # Spatial extent + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + + stats["spatial_extent_x"] = float(max_coords[0] - min_coords[0]) + stats["spatial_extent_y"] = float(max_coords[1] - min_coords[1]) + stats["centroid_x"] = float(spatial_coords[:, 0].mean()) + stats["centroid_y"] = float(spatial_coords[:, 1].mean()) + + if "cell_density" in stats: + print( + f" Global stats: cell_density={stats['cell_density']:.4f}", + flush=True, + ) + + return stats + + +def main(par): + print(f"\n>>> Reading MuData from '{par['input']}'...", flush=True) + mdata = mu.read_h5mu(par["input"]) + print(mdata, flush=True) + + print(f"\n>>> Extracting modality '{par['modality']}'...", flush=True) + if par["modality"] not in mdata.mod: + raise KeyError( + f"Modality '{par['modality']}' not found in MuData. " + f"Available modalities: {list(mdata.mod.keys())}" + ) + adata = mdata[par["modality"]] + print(adata, flush=True) + + print( + f"\n>>> Extracting spatial coordinates from .obsm['{par['obsm_spatial_coordinates']}']...", + flush=True, + ) + if par["obsm_spatial_coordinates"] not in adata.obsm: + raise KeyError( + f"Spatial key '{par['obsm_spatial_coordinates']}' not found in .obsm. " + f"Available keys: {list(adata.obsm.keys())}" + ) + + spatial_coords = adata.obsm[par["obsm_spatial_coordinates"]] + if spatial_coords.shape[1] != 2: + raise ValueError( + f"Expected 2D spatial coordinates, got shape {spatial_coords.shape}" + ) + print(f" Shape: {spatial_coords.shape} (n_cells × 2)", flush=True) + + prefix = par["output_prefix"] + + # Calculate morphology metrics + print("\n>>> Calculating morphology metrics...", flush=True) + calculate_morphology_metrics(adata, prefix) + + # Calculate position features + print("\n>>> Calculating position-based features...", flush=True) + calculate_position_features(adata, spatial_coords, prefix) + + # Calculate density metrics + print("\n>>> Calculating density metrics...", flush=True) + calculate_density_metrics( + adata, + spatial_coords, + par["density_bandwidth"], + prefix, + ) + + # Calculate Voronoi tessellation + print("\n>>> Calculating Voronoi tessellation...", flush=True) + calculate_voronoi_metrics(adata, spatial_coords, prefix) + + # Calculate global statistics + print("\n>>> Calculating global spatial statistics...", flush=True) + global_stats = calculate_global_statistics(adata, spatial_coords, par) + + # Store in uns + if "spatial_stats" not in adata.uns: + adata.uns["spatial_stats"] = {} + adata.uns["spatial_stats"].update(global_stats) + + print(f"\n>>> Writing output to '{par['output']}'...", flush=True) + mdata.write_h5mu(par["output"]) + + print("\n>>> Done!\n", flush=True) + + +if __name__ == "__main__": + sys.exit(main(par)) diff --git a/src/feature_annotation/xenium_spatial_statistics/test.py b/src/feature_annotation/xenium_spatial_statistics/test.py new file mode 100644 index 0000000..e24212a --- /dev/null +++ b/src/feature_annotation/xenium_spatial_statistics/test.py @@ -0,0 +1,75 @@ +import mudata as mu +import sys +import pytest + +## VIASH START +meta = { + "resources_dir": "resources_test/xenium", + "executable": "./xenium_spatial_statistics", +} +## VIASH END + + +def test_calculate_spatial_statistics(run_component, tmp_path): + input_path = meta["resources_dir"] + "/xenium_tiny.qc.neighbors.h5mu" + output_path = tmp_path / "output.h5mu" + + # Run the component directly on the input file + run_component( + [ + "--input", + str(input_path), + "--output", + str(output_path), + "--modality", + "rna", + "--obsm_spatial_coordinates", + "spatial", + "--density_bandwidth", + "50.0", + "--calculate_ripley_l", + "false", + ] + ) + + assert output_path.exists(), "Output file not created" + + mdata_out = mu.read_h5mu(output_path) + adata_out = mdata_out.mod["rna"] + + # Check for expected columns in .obs + expected_cols = [ + "spatial_nucleus_cell_ratio", + "spatial_cell_area_percentile", + "spatial_distance_to_centroid", + "spatial_norm_x", + "spatial_norm_y", + "spatial_kernel_density", + "spatial_graph_degree", + "spatial_voronoi_area", + "spatial_voronoi_neighbors", + ] + + for col in expected_cols: + assert col in adata_out.obs.columns, f"Column {col} missing from output .obs" + + # Check for expected metrics in .uns['spatial_stats'] + assert "spatial_stats" in adata_out.uns, "spatial_stats missing from .uns" + stats = adata_out.uns["spatial_stats"] + + expected_stats = [ + "cell_density", + "total_area", + "n_cells", + "spatial_extent_x", + "spatial_extent_y", + "centroid_x", + "centroid_y", + ] + + for stat in expected_stats: + assert stat in stats, f"Stat {stat} missing from spatial_stats" + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/src/neighbors/join_graphs/config.vsh.yaml b/src/neighbors/join_graphs/config.vsh.yaml new file mode 100644 index 0000000..a56e9a1 --- /dev/null +++ b/src/neighbors/join_graphs/config.vsh.yaml @@ -0,0 +1,91 @@ +name: join_graphs +namespace: neighbors +description: | + Combine spatial and expression neighborhood graphs into a single graph for + use in downstream clustering or trajectory analysis. + + The fusion is a linear combination of the expression connectivities + and spatial connectivities matrices, weighted by `alpha`. + + $C_{joint} = (1 - \alpha) \cdot C_{expression} + \alpha \cdot C_{spatial}$ + +authors: + - __merge__: /src/authors/jakub_majercik.yaml + roles: [ maintainer ] + +argument_groups: + - name: "Inputs" + arguments: + - name: "--input" + type: file + required: true + description: Input H5MU file with pre-computed expression and spatial neighborhood graphs. + - name: "--modality" + type: string + default: "rna" + required: false + description: Modality from the input MuData file to process. + - name: "--input_obsp_expression_graph" + type: string + default: "connectivities" + description: | + Key in `adata.obsp` containing the expression connectivity matrix. + - name: "--input_obsp_spatial_graph" + type: string + default: "spatial_connectivities" + description: | + Key in `adata.obsp` containing the spatial connectivity matrix. + + - name: "Parameters" + arguments: + - name: "--alpha" + type: double + default: 0.5 + min: 0.0 + max: 1.0 + description: | + Weight of the spatial graph in the connection fusion. + A value of 0 results in the original expression graph; + a value of 1 results in the spatial graph only. + + - name: Outputs + arguments: + - name: "--output" + type: file + direction: output + required: true + description: Output H5MU file path. + example: output.h5mu + - name: "--output_obsp_graph" + type: string + default: "spatial_expression_connectivities" + description: | + Key under which to store the fused connectivity matrix in `adata.obsp`. + __merge__: [., /src/base/h5_compression_argument.yaml] + +resources: + - type: python_script + path: script.py + - path: /src/utils/setup_logger.py + +test_resources: + - type: python_script + path: test.py + - path: /resources_test/xenium/xenium_tiny.qc.all_neighbors.pca.h5mu + +engines: + - type: docker + image: python:3.12-slim + setup: + - type: apt + packages: + - procps + - type: python + __merge__: [ /src/base/requirements/scanpy.yaml, /src/base/requirements/anndata_mudata.yaml, . ] + __merge__: [ /src/base/requirements/python_test_setup.yaml, .] + +runners: + - type: executable + - type: nextflow + directives: + label: [midcpu, midmem, middisk] diff --git a/src/neighbors/join_graphs/script.py b/src/neighbors/join_graphs/script.py new file mode 100644 index 0000000..e33594a --- /dev/null +++ b/src/neighbors/join_graphs/script.py @@ -0,0 +1,63 @@ +import sys +import mudata as mu + +## VIASH START +par = { + # Inputs + "input": "resources_test/xenium/xenium_tiny.spatial_expression_neighbors.h5mu", + "modality": "rna", + "input_obsp_expression_graph": "connectivities", + "input_obsp_spatial_graph": "spatial_connectivities", + # Fusion options + "alpha": 0.2, + # Outputs + "output": "foo.h5mu", + "output_compression": None, + "output_obsp_graph": "spatial_expression_connectivities", +} +meta = {"resources_dir": "src/utils/"} +## VIASH END + +sys.path.append(meta["resources_dir"]) +from setup_logger import setup_logger + +logger = setup_logger() + +## Read data +logger.info("Reading input data...") +adata = mu.read_h5ad(par["input"], mod=par["modality"]) + +## Validate inputs +spatial_key = par["input_obsp_spatial_graph"] +if spatial_key not in adata.obsp: + raise ValueError(f"Spatial graph key '{spatial_key}' not found in .obsp.") + +expr_key = par["input_obsp_expression_graph"] +if expr_key not in adata.obsp: + raise ValueError(f"Expression graph key '{expr_key}' not found in .obsp.") + +nn_graph_genes = adata.obsp[expr_key] +nn_graph_space = adata.obsp[spatial_key] + +## Combine graphs +alpha = par["alpha"] +logger.info( + f"Combining graphs (alpha={alpha}: spatial weight, " + f"{1 - alpha:.2f}: expression weight)..." +) +joint_graph = (1 - alpha) * nn_graph_genes + alpha * nn_graph_space +out_key = par["output_obsp_graph"] +logger.info(f"Storing result in .obsp['{out_key}']...") +adata.obsp[out_key] = joint_graph +adata.uns[out_key] = { + "params": {"alpha": alpha}, + "inputs": { + "expression_graph": expr_key, + "spatial_graph": spatial_key, + }, +} + +## Write output +logger.info("Saving output data...") +mdata = mu.MuData({par["modality"]: adata}) +mdata.write_h5mu(par["output"], compression=par["output_compression"]) diff --git a/src/neighbors/join_graphs/test.py b/src/neighbors/join_graphs/test.py new file mode 100644 index 0000000..ced86f6 --- /dev/null +++ b/src/neighbors/join_graphs/test.py @@ -0,0 +1,103 @@ +import pytest +import mudata as mu +import sys + +## VIASH START +meta = { + "executable": "./target/executable/neighbors/join_graphs/join_graphs", + "resources_dir": "resources_test/xenium/", +} +## VIASH END + +input_file = f"{meta['resources_dir']}/xenium_tiny.qc.all_neighbors.pca.h5mu" + + +def test_default_execution(run_component, tmp_path): + output = tmp_path / "fused_neighbors.h5mu" + + run_component( + [ + "--input", + input_file, + "--output", + str(output), + "--output_obsp_graph", + "fused_conn", + "--alpha", + "0.2", + "--output_compression", + "gzip", + ] + ) + + assert output.is_file(), "Output file was not created." + mdata = mu.read_h5mu(output) + assert "rna" in mdata.mod, "Expected 'rna' modality in output." + adata = mdata.mod["rna"] + + assert "fused_conn" in adata.obsp.keys(), ( + "Expected output obsp connectivities key to be present." + ) + assert "fused_conn" in adata.uns.keys(), "Expected metadata for fusion in .uns." + assert adata.uns["fused_conn"]["params"]["alpha"] == 0.2, ( + "Expected alpha parameter to be stored in .uns." + ) + + # Check graph properties (should be combination of input graphs) + fused = adata.obsp["fused_conn"] + spatial = adata.obsp["spatial_connectivities"] + expr = adata.obsp["connectivities"] + + assert fused.shape == expr.shape == spatial.shape, ( + "Fused graph should have same shape as input graphs." + ) + + +def test_alpha_zero_equals_expression(run_component, tmp_path): + output = tmp_path / "alpha_zero.h5mu" + run_component( + [ + "--input", + input_file, + "--output", + str(output), + "--alpha", + "0.0", + "--output_obsp_graph", + "fused_zero", + ] + ) + mdata = mu.read_h5mu(output) + adata = mdata.mod["rna"] + + fused = adata.obsp["fused_zero"] + # Should equal expression graph exactly (dense check might be heavy, check nnz or sum) + expr = adata.obsp["connectivities"] + assert abs(fused.sum() - expr.sum()) < 1e-6 + + +def test_alpha_one_equals_spatial(run_component, tmp_path): + output = tmp_path / "alpha_one.h5mu" + run_component( + [ + "--input", + input_file, + "--output", + str(output), + "--alpha", + "1.0", + "--output_obsp_graph", + "fused_one", + ] + ) + mdata = mu.read_h5mu(output) + adata = mdata.mod["rna"] + + fused = adata.obsp["fused_one"] + # Should equal spatial graph exactly + spatial = adata.obsp["spatial_connectivities"] + assert abs(fused.sum() - spatial.sum()) < 1e-6 + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__])) diff --git a/src/nichecompass/nichecompass/config.vsh.yaml b/src/nichecompass/nichecompass/config.vsh.yaml index a206b5c..1684bab 100644 --- a/src/nichecompass/nichecompass/config.vsh.yaml +++ b/src/nichecompass/nichecompass/config.vsh.yaml @@ -33,6 +33,12 @@ argument_groups: type: string required: false description: "Input layer to use. If None, X is used" + - name: "--var_input" + type: string + default: "filter_with_hvg" + description: | + Key in adata.var that indicates which features to include as input for the NicheCompass model. + If not provided, all features will be included. - name: "--input_obsp_spatial_connectivities" type: string default: "spatial_connectivities" @@ -429,6 +435,7 @@ resources: - type: python_script path: script.py - path: /src/utils/setup_logger.py + - path: /src/utils/subset_vars.py test_resources: - type: python_script diff --git a/src/nichecompass/nichecompass/script.py b/src/nichecompass/nichecompass/script.py index 4f0774f..4c33f78 100644 --- a/src/nichecompass/nichecompass/script.py +++ b/src/nichecompass/nichecompass/script.py @@ -15,6 +15,7 @@ par = { "input_gp_mask": "resources_test/niche/prior_knowledge_gp_mask.json", "input_obs_covariates": None, "input_obsp_spatial_connectivities": "spatial_connectivities", + "var_input": "filter_with_hvg", ## GP Mask "min_genes_per_gp": 2, "min_source_genes_per_gp": 1, @@ -89,6 +90,7 @@ meta = {"resources_dir": "src/utils/"} sys.path.append(meta["resources_dir"]) from setup_logger import setup_logger +from subset_vars import subset_vars logger = setup_logger() @@ -101,6 +103,11 @@ logger.info(f"GPU count: {torch.cuda.device_count()}") ## Read in data adata = mu.read_h5ad(par["input"], mod=par["modality"]) +# Subset to HVG +if par["var_input"]: + # Subset to HVG + adata = subset_vars(adata, subset_col=par["var_input"]).copy() + # Counts need to be float32 to be processed by nichecompass model # See https://discuss.pytorch.org/t/runtimeerror-mat1-and-mat2-must-have-the-same-dtype/166759 counts_dtype = ( diff --git a/src/nichecompass/nichecompass/test.py b/src/nichecompass/nichecompass/test.py index 6bed03f..4d40abe 100644 --- a/src/nichecompass/nichecompass/test.py +++ b/src/nichecompass/nichecompass/test.py @@ -1,5 +1,6 @@ import pytest import mudata as mu +import numpy as np import sys ## VIASH START @@ -8,21 +9,38 @@ meta = { } ## VIASH END -input_xenium = f"{meta['resources_dir']}/xenium_tiny.h5mu" -input_cosmx = f"{meta['resources_dir']}/Lung5_Rep2_tiny.h5mu" +input_path_xenium = f"{meta['resources_dir']}/xenium_tiny.h5mu" +input_path_cosmx = f"{meta['resources_dir']}/Lung5_Rep2_tiny.h5mu" gp_mask = f"{meta['resources_dir']}/prior_knowledge_gp_mask.json" -def test_simple_execution_xenium(run_component, tmp_path): +@pytest.fixture +def input_xenium_with_hvg_filter(tmp_path): + mdata = mu.read_h5mu(input_path_xenium) + adata = mdata.mod["rna"] + np.random.seed(42) + n_select = min(1000, adata.n_vars) + selected = np.random.choice(adata.n_vars, size=n_select, replace=False) + mask = np.zeros(adata.n_vars, dtype=bool) + mask[selected] = True + adata.var["filter_with_hvg"] = mask + output = tmp_path / "xenium_with_hvg_filter.h5mu" + mdata.write_h5mu(output) + return str(output) + + +def test_simple_execution_xenium(run_component, tmp_path, input_xenium_with_hvg_filter): output = tmp_path / "nc_xenium.h5mu" # run component run_component( [ "--input", - input_xenium, + input_xenium_with_hvg_filter, "--input_gp_mask", gp_mask, + "--var_input", + "filter_with_hvg", "--n_epochs", "1", "--n_epochs_all_gps", @@ -84,5 +102,48 @@ def test_simple_execution_xenium(run_component, tmp_path): ), "Expected GP targets and sources varm to have same shape" +def test_no_var_input(run_component, tmp_path, input_xenium_with_hvg_filter): + output = tmp_path / "nc_xenium_no_var_input.h5mu" + + # run component without var_input (all genes are used) + run_component( + [ + "--input", + input_xenium_with_hvg_filter, + "--input_gp_mask", + gp_mask, + "--var_input", + "", + "--n_epochs", + "1", + "--n_epochs_all_gps", + "0", + "--n_epochs_no_edge_recon", + "0", + "--n_epochs_no_cat_covariates_contrastive", + "0", + "--output", + str(output), + "--output_model", + "test_model_no_var_input", + "--output_compression", + "gzip", + ] + ) + + assert output.is_file(), "output file was not created" + mdata = mu.read_h5mu(output) + assert list(mdata.mod.keys()) == ["rna"], "Expected modality rna" + adata = mdata.mod["rna"] + + expected_uns_keys = [ + "nichecompass_gp_names", + "nichecompass_active_gp_names", + ] + assert all([uns in adata.uns.keys() for uns in expected_uns_keys]), ( + f"Expected uns keys: {expected_uns_keys}, found: {list(adata.uns.keys())}" + ) + + if __name__ == "__main__": sys.exit(pytest.main([__file__])) diff --git a/src/utils/subset_vars.py b/src/utils/subset_vars.py new file mode 100644 index 0000000..4025000 --- /dev/null +++ b/src/utils/subset_vars.py @@ -0,0 +1,49 @@ +def subset_vars(adata, subset_col): + """ + Subset AnnData object on highly variable genes or a boolean mask. + + Parameters + ---------- + adata : AnnData + Annotated data object + subset_col : str, pd.Series, pd.Index, or np.ndarray + Name of the boolean column in `adata.var` that contains the information if features should be used or not, + or a boolean mask (same length as adata.var) + + Returns + ------- + AnnData + Copy of `adata` with subsetted features + """ + import pandas as pd + import numpy as np + + # Convert all input types to a pandas Series + if isinstance(subset_col, str): + if subset_col not in adata.var.columns: + raise ValueError( + f"Requested to use .var column '{subset_col}' as a selection of genes, but the column is not available." + ) + mask = adata.var[subset_col] + elif isinstance(subset_col, pd.Series): + mask = subset_col + elif isinstance(subset_col, (pd.Index, np.ndarray, list)): + mask = pd.Series(subset_col, index=adata.var.index) + else: + raise TypeError( + "subset_col must be a string (column name) or a boolean mask (Series, Index, ndarray, or list)." + ) + + # Validate mask + if not pd.api.types.is_bool_dtype(mask): + raise ValueError( + f"Expected mask to be boolean, but found {mask.dtype}. Can not subset data." + ) + if mask.isna().sum() > 0: + raise ValueError("Mask contains NaN values. Can not subset data.") + if len(mask) != adata.n_vars: + raise ValueError( + f"Mask length {len(mask)} does not match number of variables {adata.n_vars}." + ) + + return adata[:, mask].copy() diff --git a/src/workflows/niche/nichecompass_leiden/config.vsh.yaml b/src/workflows/niche/nichecompass_leiden/config.vsh.yaml index e29575d..808e908 100644 --- a/src/workflows/niche/nichecompass_leiden/config.vsh.yaml +++ b/src/workflows/niche/nichecompass_leiden/config.vsh.yaml @@ -57,18 +57,18 @@ argument_groups: type: string default: "spatial" description: "Key in adata.obsm where spatial coordinates are stored" + - name: "--var_input" + type: string + default: "filter_with_hvg" + description: | + Key in adata.var that indicates which features to include as input for the NicheCompass model. + If not provided, all features will be included. - name: "Sample ID options" description: | Options for adding the id to .obs on the MuData object. Having a sample id present in a requirement of several components for this pipeline. arguments: - - name: "--include_sample_as_covariate" - description: | - Whether to include the sample information as a categorical covariate for the - NicheCompass model. - type: boolean - default: true - name: "--add_id_to_obs" description: "Add the value passed with --id to .obs." type: boolean @@ -357,6 +357,7 @@ dependencies: - name: neighbors/spatial_neighborhood_graph - name: nichecompass/nichecompass - name: dataflow/split_h5mu + repository: openpipeline - name: workflows/multiomics/neighbors_leiden_umap repository: openpipeline diff --git a/src/workflows/niche/nichecompass_leiden/main.nf b/src/workflows/niche/nichecompass_leiden/main.nf index 440770a..653e6d0 100644 --- a/src/workflows/niche/nichecompass_leiden/main.nf +++ b/src/workflows/niche/nichecompass_leiden/main.nf @@ -117,6 +117,7 @@ workflow run_wf { "input": state.input, "input_gp_mask": state.input_gp_mask, "input_obs_covariates": state.input_obs_covariates, + "var_input": state.var_input, "modality": state.modality, "layer": state.layer, "min_genes_per_gp": state.min_genes_per_gp, diff --git a/src/workflows/niche/nichecompass_leiden/test.nf b/src/workflows/niche/nichecompass_leiden/test.nf index 6fdb100..61d4d4d 100644 --- a/src/workflows/niche/nichecompass_leiden/test.nf +++ b/src/workflows/niche/nichecompass_leiden/test.nf @@ -19,7 +19,8 @@ workflow test_wf { n_epochs_all_gps: 0, n_epochs_no_edge_recon: 0, n_epochs_no_cat_covariates_contrastive_loss: 0, - output_model: "simple_execution_test_model" + output_model: "simple_execution_test_model", + var_input: null ] ]) | map { state -> [state.id, state] } diff --git a/target/_private/executable/filter/subset_cosmx/.config.vsh.yaml b/target/_private/executable/filter/subset_cosmx/.config.vsh.yaml index cdcd200..55f2441 100644 --- a/target/_private/executable/filter/subset_cosmx/.config.vsh.yaml +++ b/target/_private/executable/filter/subset_cosmx/.config.vsh.yaml @@ -112,7 +112,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -227,7 +227,7 @@ build_info: output: "target/_private/executable/filter/subset_cosmx" executable: "target/_private/executable/filter/subset_cosmx/subset_cosmx" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -241,7 +241,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_private/executable/filter/subset_cosmx/subset_cosmx b/target/_private/executable/filter/subset_cosmx/subset_cosmx index 9923161..1e78282 100755 --- a/target/_private/executable/filter/subset_cosmx/subset_cosmx +++ b/target/_private/executable/filter/subset_cosmx/subset_cosmx @@ -458,9 +458,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component filter subset_cosmx" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/_private/nextflow/filter/subset_cosmx/.config.vsh.yaml b/target/_private/nextflow/filter/subset_cosmx/.config.vsh.yaml index 1804360..cb9af5c 100644 --- a/target/_private/nextflow/filter/subset_cosmx/.config.vsh.yaml +++ b/target/_private/nextflow/filter/subset_cosmx/.config.vsh.yaml @@ -112,7 +112,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -227,7 +227,7 @@ build_info: output: "target/_private/nextflow/filter/subset_cosmx" executable: "target/_private/nextflow/filter/subset_cosmx/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -241,7 +241,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_private/nextflow/filter/subset_cosmx/main.nf b/target/_private/nextflow/filter/subset_cosmx/main.nf index 791a678..0e7dc02 100644 --- a/target/_private/nextflow/filter/subset_cosmx/main.nf +++ b/target/_private/nextflow/filter/subset_cosmx/main.nf @@ -3187,7 +3187,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3333,7 +3333,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/_private/nextflow/filter/subset_cosmx", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3353,7 +3353,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml b/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml index acbe309..1a6956b 100644 --- a/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml +++ b/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml @@ -48,7 +48,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -157,7 +157,7 @@ build_info: output: "target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test" executable: "target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/spaceranger_mapping_test" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -171,7 +171,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/spaceranger_mapping_test b/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/spaceranger_mapping_test index 91a2913..28161ca 100755 --- a/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/spaceranger_mapping_test +++ b/target/_test/executable/test_workflows/ingestion/spaceranger_mapping_test/spaceranger_mapping_test @@ -459,9 +459,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component test_workflows/ingestion spaceranger_mapping_test" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:23Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml b/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml index be1b3a0..7a5a54f 100644 --- a/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml +++ b/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml @@ -47,7 +47,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -156,7 +156,7 @@ build_info: output: "target/_test/executable/test_workflows/niche/nichecompass_leiden_test" executable: "target/_test/executable/test_workflows/niche/nichecompass_leiden_test/nichecompass_leiden_test" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -170,7 +170,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/nichecompass_leiden_test b/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/nichecompass_leiden_test index 3aa810d..34eda3f 100755 --- a/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/nichecompass_leiden_test +++ b/target/_test/executable/test_workflows/niche/nichecompass_leiden_test/nichecompass_leiden_test @@ -459,9 +459,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component test_workflows/niche nichecompass_leiden_test" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:23Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml b/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml index ca81a94..b4596d1 100644 --- a/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml +++ b/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/.config.vsh.yaml @@ -48,7 +48,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -157,7 +157,7 @@ build_info: output: "target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test" executable: "target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -171,7 +171,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/main.nf b/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/main.nf index ff4996e..d362f27 100644 --- a/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/main.nf +++ b/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test/main.nf @@ -3104,7 +3104,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3235,7 +3235,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/_test/nextflow/test_workflows/ingestion/spaceranger_mapping_test", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3255,7 +3255,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml b/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml index 367ef80..3080d31 100644 --- a/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml +++ b/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/.config.vsh.yaml @@ -47,7 +47,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -156,7 +156,7 @@ build_info: output: "target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test" executable: "target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -170,7 +170,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/main.nf b/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/main.nf index e5fde77..fc1a421 100644 --- a/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/main.nf +++ b/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test/main.nf @@ -3104,7 +3104,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3235,7 +3235,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/_test/nextflow/test_workflows/niche/nichecompass_leiden_test", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3255,7 +3255,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml index 3d4b2af..3f6fe1d 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "split_modalities" namespace: "workflows/multiomics" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -183,13 +183,13 @@ build_info: output: "target/_private/nextflow/workflows/multiomics/split_modalities" executable: "target/_private/nextflow/workflows/multiomics/split_modalities/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/dataflow/split_modalities" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -219,7 +219,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/main.nf index e1ba422..4734ad5 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/main.nf @@ -1,4 +1,4 @@ -// split_modalities v4.0.2 +// split_modalities v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "split_modalities", "namespace" : "workflows/multiomics", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3274,12 +3274,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/_private/nextflow/workflows/multiomics/split_modalities", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3304,7 +3304,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config index 14e373d..958378b 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/split_modalities' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'A pipeline to split a multimodal mudata files into several unimodal mudata files.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/multiomics/split_modalities/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml index e0c2fda..019d7e6 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "log_normalize" namespace: "workflows/rna" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -198,7 +198,7 @@ build_info: output: "target/_private/nextflow/workflows/rna/log_normalize" executable: "target/_private/nextflow/workflows/rna/log_normalize/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/transform/normalize_total" @@ -206,7 +206,7 @@ build_info: - "target/nextflow/transform/delete_layer" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -236,7 +236,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/main.nf index d2e13de..348a70b 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/main.nf @@ -1,4 +1,4 @@ -// log_normalize v4.0.2 +// log_normalize v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "log_normalize", "namespace" : "workflows/rna", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3294,12 +3294,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/_private/nextflow/workflows/rna/log_normalize", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3324,7 +3324,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/nextflow.config index d3ea299..51c0bc3 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/rna/log_normalize' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Performs normalization and subsequent log-transformation of raw count data.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/_private/nextflow/workflows/rna/log_normalize/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/_private/nextflow/workflows/rna/log_normalize/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/.config.vsh.yaml index e9d7e49..ee5f27d 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "leiden" namespace: "cluster" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -215,7 +215,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -258,11 +258,11 @@ build_info: output: "target/nextflow/cluster/leiden" executable: "target/nextflow/cluster/leiden/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -292,7 +292,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/main.nf index 6a5f549..c2c32f8 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/main.nf @@ -1,4 +1,4 @@ -// leiden v4.0.2 +// leiden v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "leiden", "namespace" : "cluster", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3294,7 +3294,7 @@ meta = [ "id" : "docker", "image" : "python:3.13-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3352,12 +3352,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/cluster/leiden", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3382,7 +3382,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4147,7 +4147,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/cluster/leiden", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "highcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow.config index bb4c68d..e2ff599 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'cluster/leiden' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Cluster cells using the [Leiden algorithm] [Traag18] implemented in the [Scanpy framework] [Wolf18]. \nLeiden is an improved version of the [Louvain algorithm] [Blondel08]. \nIt has been proposed for single-cell analysis by [Levine15] [Levine15]. \nThis requires having ran `neighbors/find_neighbors` or `neighbors/bbknn` first.\n\n[Blondel08]: Blondel et al. (2008), Fast unfolding of communities in large networks, J. Stat. Mech. \n[Levine15]: Levine et al. (2015), Data-Driven Phenotypic Dissection of AML Reveals Progenitor-like Cells that Correlate with Prognosis, Cell. \n[Traag18]: Traag et al. (2018), From Louvain to Leiden: guaranteeing well-connected communities arXiv. \n[Wolf18]: Wolf et al. (2018), Scanpy: large-scale single-cell gene expression data analysis, Genome Biology. \n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/cluster/leiden/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/cluster/leiden/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml index 05228f2..fc0d331 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "concatenate_h5mu" namespace: "dataflow" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -253,7 +253,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -300,11 +300,11 @@ build_info: output: "target/nextflow/dataflow/concatenate_h5mu" executable: "target/nextflow/dataflow/concatenate_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -334,7 +334,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/main.nf index c12b5cd..18b9dd3 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/main.nf @@ -1,4 +1,4 @@ -// concatenate_h5mu v4.0.2 +// concatenate_h5mu v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "concatenate_h5mu", "namespace" : "dataflow", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3328,7 +3328,7 @@ meta = [ "id" : "docker", "image" : "python:3.13-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3393,12 +3393,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dataflow/concatenate_h5mu", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3423,7 +3423,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4266,7 +4266,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/dataflow/concatenate_h5mu", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "midcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow.config index c716b1f..0ba8d92 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/concatenate_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Concatenate observations from samples in several (uni- and/or multi-modal) MuData files into a single file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/.config.vsh.yaml index 0cfdabb..59fbb19 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "merge" namespace: "dataflow" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -163,7 +163,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -204,11 +204,11 @@ build_info: output: "target/nextflow/dataflow/merge" executable: "target/nextflow/dataflow/merge/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -238,7 +238,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/main.nf index fe4ec0a..5524946 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/main.nf @@ -1,4 +1,4 @@ -// merge v4.0.2 +// merge v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "merge", "namespace" : "dataflow", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3246,7 +3246,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3302,12 +3302,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dataflow/merge", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3332,7 +3332,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3847,7 +3847,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/dataflow/merge", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow.config index f4d9aec..9c85150 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/merge' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Combine one or more single-modality .h5mu files together into one .h5mu file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/merge/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/merge/setup_logger.py diff --git a/target/nextflow/dataflow/split_h5mu/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/.config.vsh.yaml similarity index 81% rename from target/nextflow/dataflow/split_h5mu/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/.config.vsh.yaml index 1d020e8..58cc0c2 100644 --- a/target/nextflow/dataflow/split_h5mu/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "split_h5mu" namespace: "dataflow" -version: "niche-compass" +version: "v4.0.3" authors: - name: "Dorien Roosen" roles: @@ -123,13 +123,9 @@ status: "enabled" scope: image: "public" target: "public" -repositories: -- type: "vsh" - name: "openpipeline" - repo: "openpipeline" - tag: "v4.0.2" +license: "MIT" links: - repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" runners: - type: "executable" @@ -207,7 +203,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "niche-compass" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -233,7 +229,7 @@ engines: - type: "python" user: false packages: - - "viashpy==0.9.0" + - "viashpy==0.8.0" github: - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" upgrade: true @@ -248,21 +244,31 @@ build_info: output: "target/nextflow/dataflow/split_h5mu" executable: "target/nextflow/dataflow/split_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" - git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" + git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: - name: "openpipeline_spatial" - version: "niche-compass" + name: "openpipeline" + version: "v4.0.3" + summary: "Best-practice workflows for single-cell multi-omics analyses.\n" + description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ + \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ + \nIn terms of workflows, the following has been made available, but keep in mind\ + \ that\nindividual tools and functionality can be executed as standalone components\ + \ as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n\ + \ * Ingestion: Read mapping and generating a count matrix.\n * Single sample\ + \ processing: cell filtering and doublet detection.\n * Multisample processing:\ + \ Count transformation, normalization, QC metric calulations.\n * Integration:\ + \ Clustering, integration and batch correction using single and multimodal methods.\n\ + \ * Downstream analysis workflows\n" info: test_resources: - type: "s3" - path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + path: "s3://openpipelines-data" dest: "resources_test" - repositories: - - type: "vsh" - name: "openpipeline" - repo: "openpipeline" - tag: "v4.0.2" + nextflow_labels_ci: + - path: "src/workflows/utils/labels_ci.config" + description: "Adds the correct memory and CPU labels when running on the Viash\ + \ Hub CI." viash_version: "0.9.4" source: "src" target: "target" @@ -272,8 +278,15 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" + keywords: + - "single-cell" + - "multimodal" + license: "MIT" organization: "vsh" links: - repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + repository: "https://github.com/openpipelines-bio/openpipeline" docker_registry: "ghcr.io" + homepage: "https://openpipelines.bio" + documentation: "https://openpipelines.bio/fundamentals" + issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues" diff --git a/target/nextflow/dataflow/split_h5mu/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/main.nf similarity index 98% rename from target/nextflow/dataflow/split_h5mu/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/main.nf index b4ebd02..4b7dfff 100644 --- a/target/nextflow/dataflow/split_h5mu/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/main.nf @@ -1,4 +1,4 @@ -// split_h5mu niche-compass +// split_h5mu v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "split_h5mu", "namespace" : "dataflow", - "version" : "niche-compass", + "version" : "v4.0.3", "authors" : [ { "name" : "Dorien Roosen", @@ -3192,16 +3192,9 @@ meta = [ "image" : "public", "target" : "public" }, - "repositories" : [ - { - "type" : "vsh", - "name" : "openpipeline", - "repo" : "openpipeline", - "tag" : "v4.0.2" - } - ], + "license" : "MIT", "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "repository" : "https://github.com/openpipelines-bio/openpipeline", "docker_registry" : "ghcr.io" }, "runners" : [ @@ -3292,7 +3285,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "niche-compass", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3328,7 +3321,7 @@ meta = [ "type" : "python", "user" : false, "packages" : [ - "viashpy==0.9.0" + "viashpy==0.8.0" ], "github" : [ "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" @@ -3348,29 +3341,29 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dataflow/split_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", - "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", + "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { - "name" : "openpipeline_spatial", - "version" : "niche-compass", + "name" : "openpipeline", + "version" : "v4.0.3", + "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", + "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { "test_resources" : [ { "type" : "s3", - "path" : "s3://openpipelines-bio/openpipeline_spatial/resources_test", + "path" : "s3://openpipelines-data", "dest" : "resources_test" } + ], + "nextflow_labels_ci" : [ + { + "path" : "src/workflows/utils/labels_ci.config", + "description" : "Adds the correct memory and CPU labels when running on the Viash Hub CI." + } ] }, - "repositories" : [ - { - "type" : "vsh", - "name" : "openpipeline", - "repo" : "openpipeline", - "tag" : "v4.0.2" - } - ], "viash_version" : "0.9.4", "source" : "/workdir/root/repo/src", "target" : "/workdir/root/repo/target", @@ -3378,12 +3371,20 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], + "keywords" : [ + "single-cell", + "multimodal" + ], + "license" : "MIT", "organization" : "vsh", "links" : { - "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", - "docker_registry" : "ghcr.io" + "repository" : "https://github.com/openpipelines-bio/openpipeline", + "docker_registry" : "ghcr.io", + "homepage" : "https://openpipelines.bio", + "documentation" : "https://openpipelines.bio/fundamentals", + "issue_tracker" : "https://github.com/openpipelines-bio/openpipeline/issues" } } }''')) @@ -3908,8 +3909,8 @@ meta["defaults"] = [ directives: readJsonBlob('''{ "container" : { "registry" : "images.viash-hub.com", - "image" : "vsh/openpipeline_spatial/dataflow/split_h5mu", - "tag" : "niche-compass" + "image" : "vsh/openpipeline/dataflow/split_h5mu", + "tag" : "v4.0.3" }, "label" : [ "lowcpu", diff --git a/target/nextflow/dataflow/split_h5mu/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow.config similarity index 99% rename from target/nextflow/dataflow/split_h5mu/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow.config index 47ec54a..f9970f8 100644 --- a/target/nextflow/dataflow/split_h5mu/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/split_h5mu' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'niche-compass' + version = 'v4.0.3' description = 'Split the samples of a single modality from a .h5mu (multimodal) sample into seperate .h5mu files based on the values of an .obs column of this modality. \n' author = 'Dorien Roosen' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow_labels.config diff --git a/target/nextflow/dataflow/split_h5mu/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow_schema.json similarity index 100% rename from target/nextflow/dataflow/split_h5mu/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/.config.vsh.yaml index 7b551c7..ea7bd80 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "split_modalities" namespace: "dataflow" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -190,7 +190,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -231,11 +231,11 @@ build_info: output: "target/nextflow/dataflow/split_modalities" executable: "target/nextflow/dataflow/split_modalities/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -265,7 +265,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/main.nf index 0fb4023..9446a6e 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/main.nf @@ -1,4 +1,4 @@ -// split_modalities v4.0.2 +// split_modalities v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "split_modalities", "namespace" : "dataflow", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3280,7 +3280,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3336,12 +3336,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dataflow/split_modalities", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3366,7 +3366,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3865,7 +3865,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/dataflow/split_modalities", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow.config index 49fb3fb..50808ba 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dataflow/split_modalities' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Split the modalities from a single .h5mu multimodal sample into seperate .h5mu files. \n' author = 'Dries Schaumont, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/split_modalities/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_modalities/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/.config.vsh.yaml index da49ee6..01250b1 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "pca" namespace: "dimred" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -240,7 +240,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -276,11 +276,11 @@ build_info: output: "target/nextflow/dimred/pca" executable: "target/nextflow/dimred/pca/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -310,7 +310,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/main.nf index 673a497..51366e4 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/main.nf @@ -1,4 +1,4 @@ -// pca v4.0.2 +// pca v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "pca", "namespace" : "dimred", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3333,7 +3333,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3380,12 +3380,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dimred/pca", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3410,7 +3410,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3933,7 +3933,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/dimred/pca", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "highcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow.config index 4018099..dfd1032 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dimred/pca' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Computes PCA coordinates, loadings and variance decomposition. Uses the implementation of scikit-learn [Pedregosa11].\n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/pca/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/pca/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/.config.vsh.yaml index fb378b9..8959aa4 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "umap" namespace: "dimred" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -294,7 +294,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -330,11 +330,11 @@ build_info: output: "target/nextflow/dimred/umap" executable: "target/nextflow/dimred/umap/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -364,7 +364,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/main.nf index 02a6940..4641b53 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/main.nf @@ -1,4 +1,4 @@ -// umap v4.0.2 +// umap v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "umap", "namespace" : "dimred", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3382,7 +3382,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3429,12 +3429,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/dimred/umap", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3459,7 +3459,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3971,7 +3971,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/dimred/umap", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "highcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow.config index bc69bab..344a788 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'dimred/umap' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'UMAP (Uniform Manifold Approximation and Projection) is a manifold learning technique suitable for visualizing high-dimensional data. Besides tending to be faster than tSNE, it optimizes the embedding such that it best reflects the topology of the data, which we represent throughout Scanpy using a neighborhood graph. tSNE, by contrast, optimizes the distribution of nearest-neighbor distances in the embedding such that these best match the distribution of distances in the high-dimensional space. We use the implementation of umap-learn [McInnes18]. For a few comparisons of UMAP with tSNE, see this preprint.\n' author = 'Dries De Maeyer' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dimred/umap/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dimred/umap/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml index 6985259..ffd21ae 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "highly_variable_features_scanpy" namespace: "feature_annotation" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -342,7 +342,7 @@ engines: id: "docker" image: "python:3.12" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "python" @@ -381,11 +381,11 @@ build_info: output: "target/nextflow/feature_annotation/highly_variable_features_scanpy" executable: "target/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -415,7 +415,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf index e196e46..c9c429f 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/main.nf @@ -1,4 +1,4 @@ -// highly_variable_features_scanpy v4.0.2 +// highly_variable_features_scanpy v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "highly_variable_features_scanpy", "namespace" : "feature_annotation", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3430,7 +3430,7 @@ meta = [ "id" : "docker", "image" : "python:3.12", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3481,12 +3481,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/feature_annotation/highly_variable_features_scanpy", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3511,7 +3511,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4110,7 +4110,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/feature_annotation/highly_variable_features_scanpy", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config index 7195163..b239120 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'feature_annotation/highly_variable_features_scanpy' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Annotate highly variable features [Satija15] [Zheng17] [Stuart19].\n\nExpects logarithmized data, except when flavor=\'seurat_v3\' in which count data is expected.\n\nDepending on flavor, this reproduces the R-implementations of Seurat [Satija15], Cell Ranger [Zheng17], and Seurat v3 [Stuart19].\n\nFor the dispersion-based methods ([Satija15] and [Zheng17]), the normalized dispersion is obtained by scaling with the mean and standard deviation of the dispersions for features falling into a given bin for mean expression of features. This means that for each bin of mean expression, highly variable features are selected.\n\nFor [Stuart19], a normalized variance for each feature is computed. First, the data are standardized (i.e., z-score normalization per feature) with a regularized standard deviation. Next, the normalized variance is computed as the variance of each feature after the transformation. Features are ranked by the normalized variance.\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/feature_annotation/highly_variable_features_scanpy/subset_vars.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/.config.vsh.yaml index 1f2d5f9..87ca907 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "delimit_fraction" namespace: "filter" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -219,7 +219,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -260,11 +260,11 @@ build_info: output: "target/nextflow/filter/delimit_fraction" executable: "target/nextflow/filter/delimit_fraction/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -294,7 +294,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/main.nf index cd48505..24403ac 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/main.nf @@ -1,4 +1,4 @@ -// delimit_fraction v4.0.2 +// delimit_fraction v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "delimit_fraction", "namespace" : "filter", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3312,7 +3312,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3368,12 +3368,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/filter/delimit_fraction", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3398,7 +3398,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3928,7 +3928,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/filter/delimit_fraction", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow.config index 940bbae..c8eb844 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/delimit_fraction' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Turns a column containing values between 0 and 1 into a boolean column based on thresholds.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/delimit_fraction/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/delimit_fraction/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/.config.vsh.yaml index c87bbf7..46739d7 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "do_filter" namespace: "filter" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Robrecht Cannoodt" roles: @@ -193,7 +193,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -228,11 +228,11 @@ build_info: output: "target/nextflow/filter/do_filter" executable: "target/nextflow/filter/do_filter/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -262,7 +262,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/main.nf index 6edcbde..56666e3 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/main.nf @@ -1,4 +1,4 @@ -// do_filter v4.0.2 +// do_filter v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "do_filter", "namespace" : "filter", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Robrecht Cannoodt", @@ -3282,7 +3282,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3328,12 +3328,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/filter/do_filter", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3358,7 +3358,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3851,7 +3851,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/filter/do_filter", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow.config index 461bcba..296eff8 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/do_filter' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Remove observations and variables based on specified .obs and .var columns.\n' author = 'Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/do_filter/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/do_filter/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/.config.vsh.yaml index 50aed5a..43648be 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "filter_with_counts" namespace: "filter" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -280,7 +280,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -321,11 +321,11 @@ build_info: output: "target/nextflow/filter/filter_with_counts" executable: "target/nextflow/filter/filter_with_counts/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -355,7 +355,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/main.nf index 46b2812..447e900 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/main.nf @@ -1,4 +1,4 @@ -// filter_with_counts v4.0.2 +// filter_with_counts v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "filter_with_counts", "namespace" : "filter", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3392,7 +3392,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3448,12 +3448,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/filter/filter_with_counts", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3478,7 +3478,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4050,7 +4050,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/filter/filter_with_counts", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow.config index 1845720..a522643 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/filter_with_counts' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Filter scRNA-seq data based on the primary QC metrics. \nThis is based on both the UMI counts, the gene counts \nand the mitochondrial genes (genes starting with mt/MT).\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_counts/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_counts/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/.config.vsh.yaml index 3b33f09..8aa6b02 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "filter_with_scrublet" namespace: "filter" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -333,7 +333,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -378,11 +378,11 @@ build_info: output: "target/nextflow/filter/filter_with_scrublet" executable: "target/nextflow/filter/filter_with_scrublet/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -412,7 +412,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/main.nf index cf79f99..8741a30 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/main.nf @@ -1,4 +1,4 @@ -// filter_with_scrublet v4.0.2 +// filter_with_scrublet v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "filter_with_scrublet", "namespace" : "filter", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3429,7 +3429,7 @@ meta = [ "id" : "docker", "image" : "python:3.13-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3489,12 +3489,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/filter/filter_with_scrublet", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3519,7 +3519,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4065,7 +4065,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/filter/filter_with_scrublet", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "highcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow.config index 533aa1c..23ab7fe 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'filter/filter_with_scrublet' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Doublet detection using the Scrublet method (Wolock, Lopez and Klein, 2019).\nThe method tests for potential doublets by using the expression profiles of\ncells to generate synthetic potential doubles which are tested against cells. \nThe method returns a "doublet score" on which it calls for potential doublets.\n\nFor the source code please visit https://github.com/AllonKleinLab/scrublet.\n\nFor 10x we expect the doublet rates to be:\n Multiplet Rate (%) - # of Cells Loaded - # of Cells Recovered\n ~0.4% ~800 ~500\n ~0.8% ~1,600 ~1,000\n ~1.6% ~3,200 ~2,000\n ~2.3% ~4,800 ~3,000\n ~3.1% ~6,400 ~4,000\n ~3.9% ~8,000 ~5,000\n ~4.6% ~9,600 ~6,000\n ~5.4% ~11,200 ~7,000\n ~6.1% ~12,800 ~8,000\n ~6.9% ~14,400 ~9,000\n ~7.6% ~16,000 ~10,000\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/filter/filter_with_scrublet/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/filter/filter_with_scrublet/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/.config.vsh.yaml index 19b5bf3..1ae5e86 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "add_id" namespace: "metadata" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -185,7 +185,7 @@ engines: id: "docker" image: "python:3.11-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -226,11 +226,11 @@ build_info: output: "target/nextflow/metadata/add_id" executable: "target/nextflow/metadata/add_id/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -260,7 +260,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/main.nf index 7b086f0..d280c89 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/main.nf @@ -1,4 +1,4 @@ -// add_id v4.0.2 +// add_id v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "add_id", "namespace" : "metadata", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3269,7 +3269,7 @@ meta = [ "id" : "docker", "image" : "python:3.11-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3325,12 +3325,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/metadata/add_id", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3355,7 +3355,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3871,7 +3871,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/metadata/add_id", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow.config index 3360141..16c4363 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'metadata/add_id' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Add id of .obs. Also allows to make .obs_names (the .obs index) unique \nby prefixing the values with an unique id per .h5mu file.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/add_id/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/add_id/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/.config.vsh.yaml index 3f828b2..2017432 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "grep_annotation_column" namespace: "metadata" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -241,7 +241,7 @@ engines: id: "docker" image: "python:3.11-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -282,11 +282,11 @@ build_info: output: "target/nextflow/metadata/grep_annotation_column" executable: "target/nextflow/metadata/grep_annotation_column/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -316,7 +316,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/main.nf index c22fa53..4133cfa 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/main.nf @@ -1,4 +1,4 @@ -// grep_annotation_column v4.0.2 +// grep_annotation_column v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "grep_annotation_column", "namespace" : "metadata", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3335,7 +3335,7 @@ meta = [ "id" : "docker", "image" : "python:3.11-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3391,12 +3391,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/metadata/grep_annotation_column", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3421,7 +3421,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4010,7 +4010,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/metadata/grep_annotation_column", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow.config index 08372d4..f80179a 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'metadata/grep_annotation_column' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Perform a regex lookup on a column from the annotation matrices .obs or .var.\nThe annotation matrix can originate from either a modality, or all modalities (global .var or .obs).\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/grep_annotation_column/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/grep_annotation_column/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml index 0981699..b04b8ad 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "move_obsm_to_obs" namespace: "metadata" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -183,7 +183,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -224,11 +224,11 @@ build_info: output: "target/nextflow/metadata/move_obsm_to_obs" executable: "target/nextflow/metadata/move_obsm_to_obs/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -258,7 +258,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/main.nf index fa0b5ae..a7ec6bf 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/main.nf @@ -1,4 +1,4 @@ -// move_obsm_to_obs v4.0.2 +// move_obsm_to_obs v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "move_obsm_to_obs", "namespace" : "metadata", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3268,7 +3268,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3324,12 +3324,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/metadata/move_obsm_to_obs", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3354,7 +3354,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3860,7 +3860,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/metadata/move_obsm_to_obs", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow.config index 99670d0..0714aad 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'metadata/move_obsm_to_obs' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Move a matrix from .obsm to .obs. Newly created columns in .obs will \nbe created from the .obsm key suffixed with an underscore and the name of the columns\nof the specified .obsm matrix.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/metadata/move_obsm_to_obs/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/.config.vsh.yaml index f8e4048..b700608 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "find_neighbors" namespace: "neighbors" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -296,7 +296,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -338,11 +338,11 @@ build_info: output: "target/nextflow/neighbors/find_neighbors" executable: "target/nextflow/neighbors/find_neighbors/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -372,7 +372,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/main.nf index ea56e8e..aee4747 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/main.nf @@ -1,4 +1,4 @@ -// find_neighbors v4.0.2 +// find_neighbors v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "find_neighbors", "namespace" : "neighbors", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3395,7 +3395,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3452,12 +3452,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/neighbors/find_neighbors", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3482,7 +3482,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3983,7 +3983,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/neighbors/find_neighbors", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "lowcpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow.config index e164931..4075c4b 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'neighbors/find_neighbors' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Compute a neighborhood graph of observations [McInnes18].\n\nThe neighbor search efficiency of this heavily relies on UMAP [McInnes18], which also provides a method for estimating connectivities of data points - the connectivity of the manifold (method==\'umap\'). If method==\'gauss\', connectivities are computed according to [Coifman05], in the adaption of [Haghverdi16].\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/neighbors/find_neighbors/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/neighbors/find_neighbors/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml index c889842..ad9e266 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "calculate_qc_metrics" namespace: "qc" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -295,7 +295,7 @@ engines: id: "docker" image: "python:3.11-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -342,11 +342,11 @@ build_info: output: "target/nextflow/qc/calculate_qc_metrics" executable: "target/nextflow/qc/calculate_qc_metrics/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -376,7 +376,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/main.nf index 16a8261..3955b29 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/main.nf @@ -1,4 +1,4 @@ -// calculate_qc_metrics v4.0.2 +// calculate_qc_metrics v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "calculate_qc_metrics", "namespace" : "qc", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3381,7 +3381,7 @@ meta = [ "id" : "docker", "image" : "python:3.11-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3446,12 +3446,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/qc/calculate_qc_metrics", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3476,7 +3476,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -4107,7 +4107,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/qc/calculate_qc_metrics", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "singlecpu", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow.config index ab6a39a..0d6eaae 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'qc/calculate_qc_metrics' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Add basic quality control metrics to an .h5mu file.\n\nThe metrics are comparable to what scanpy.pp.calculate_qc_metrics output,\nalthough they have slightly different names:\n\nVar metrics (name in this component -> name in scanpy):\n - pct_dropout -> pct_dropout_by_{expr_type}\n - num_nonzero_obs -> n_cells_by_{expr_type}\n - obs_mean -> mean_{expr_type}\n - total_counts -> total_{expr_type}\n\n Obs metrics:\n - num_nonzero_vars -> n_genes_by_{expr_type}\n - pct_{var_qc_metrics} -> pct_{expr_type}_{qc_var}\n - total_counts_{var_qc_metrics} -> total_{expr_type}_{qc_var}\n - pct_of_counts_in_top_{top_n_vars}_vars -> pct_{expr_type}_in_top_{n}_{var_type}\n - total_counts -> total_{expr_type}\n \n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/qc/calculate_qc_metrics/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/qc/calculate_qc_metrics/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/.config.vsh.yaml index f3bc6d1..0a00e4e 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "clr" namespace: "transform" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -203,7 +203,7 @@ engines: id: "docker" image: "python:3.13-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -240,11 +240,11 @@ build_info: output: "target/nextflow/transform/clr" executable: "target/nextflow/transform/clr/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -274,7 +274,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/main.nf index eff671d..8b680cd 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/main.nf @@ -1,4 +1,4 @@ -// clr v4.0.2 +// clr v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "clr", "namespace" : "transform", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3289,7 +3289,7 @@ meta = [ "id" : "docker", "image" : "python:3.13-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3337,12 +3337,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/transform/clr", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3367,7 +3367,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3853,7 +3853,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/transform/clr", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "lowmem", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow.config index 09cf62f..13c6022 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/clr' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Perform CLR normalization on CITE-seq data (Stoeckius et al., 2017).\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/clr/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/clr/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/.config.vsh.yaml similarity index 97% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/.config.vsh.yaml index 784cb41..000e6f1 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "delete_layer" namespace: "transform" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -186,7 +186,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -227,11 +227,11 @@ build_info: output: "target/nextflow/transform/delete_layer" executable: "target/nextflow/transform/delete_layer/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -261,7 +261,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/main.nf index 9014366..2c85dc8 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/main.nf @@ -1,4 +1,4 @@ -// delete_layer v4.0.2 +// delete_layer v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "delete_layer", "namespace" : "transform", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3273,7 +3273,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3329,12 +3329,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/transform/delete_layer", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3359,7 +3359,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3844,7 +3844,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/transform/delete_layer", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "midmem", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow.config index 1506cac..38424eb 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/delete_layer' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Delete an anndata layer from one or more modalities.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/delete_layer/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/delete_layer/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/.config.vsh.yaml index 2684fe8..929508b 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "log1p" namespace: "transform" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -217,7 +217,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -253,11 +253,11 @@ build_info: output: "target/nextflow/transform/log1p" executable: "target/nextflow/transform/log1p/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -287,7 +287,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/main.nf index 78f335b..92a48d2 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/main.nf @@ -1,4 +1,4 @@ -// log1p v4.0.2 +// log1p v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "log1p", "namespace" : "transform", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3316,7 +3316,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3363,12 +3363,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/transform/log1p", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3393,7 +3393,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3879,7 +3879,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/transform/log1p", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "midmem", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow.config index acc78f6..eadfa35 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/log1p' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Logarithmize the data matrix. Computes X = log(X + 1), where log denotes the natural logarithm unless a different base is given.\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/log1p/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/log1p/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/.config.vsh.yaml index 6d7e887..3563eb8 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "normalize_total" namespace: "transform" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -229,7 +229,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -271,11 +271,11 @@ build_info: output: "target/nextflow/transform/normalize_total" executable: "target/nextflow/transform/normalize_total/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -305,7 +305,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/main.nf index c1d9e98..32c29e2 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/main.nf @@ -1,4 +1,4 @@ -// normalize_total v4.0.2 +// normalize_total v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3036,7 +3036,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "normalize_total", "namespace" : "transform", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3319,7 +3319,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3376,12 +3376,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/transform/normalize_total", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3406,7 +3406,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3896,7 +3896,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/transform/normalize_total", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "midmem", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow.config index cd3f7e8..414222a 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/normalize_total' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Normalize counts per cell.\n\nNormalize each cell by total counts over all genes, so that every cell has the same total count after normalization. If choosing target_sum=1e6, this is CPM normalization.\n\nIf exclude_highly_expressed=True, very highly expressed genes are excluded from the computation of the normalization factor (size factor) for each cell. This is meaningful as these can strongly influence the resulting normalized values for all other genes [Weinreb17].\n' author = 'Dries De Maeyer, Robrecht Cannoodt' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/normalize_total/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/setup_logger.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/normalize_total/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/.config.vsh.yaml index 277f9c1..2cf0725 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "scale" namespace: "transform" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -204,7 +204,7 @@ engines: id: "docker" image: "python:3.12-slim" target_registry: "images.viash-hub.com" - target_tag: "v4.0.2" + target_tag: "v4.0.3" namespace_separator: "/" setup: - type: "apt" @@ -246,11 +246,11 @@ build_info: output: "target/nextflow/transform/scale" executable: "target/nextflow/transform/scale/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -280,7 +280,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/compress_h5mu.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/compress_h5mu.py similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/compress_h5mu.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/compress_h5mu.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/main.nf index 4c62e9e..2c1685d 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/main.nf @@ -1,4 +1,4 @@ -// scale v4.0.2 +// scale v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "scale", "namespace" : "transform", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3291,7 +3291,7 @@ meta = [ "id" : "docker", "image" : "python:3.12-slim", "target_registry" : "images.viash-hub.com", - "target_tag" : "v4.0.2", + "target_tag" : "v4.0.3", "namespace_separator" : "/", "setup" : [ { @@ -3348,12 +3348,12 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/transform/scale", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3378,7 +3378,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", @@ -3873,7 +3873,7 @@ meta["defaults"] = [ "container" : { "registry" : "images.viash-hub.com", "image" : "vsh/openpipeline/transform/scale", - "tag" : "v4.0.2" + "tag" : "v4.0.3" }, "label" : [ "lowmem", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow.config index 8c0caa7..26bd36c 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'transform/scale' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Scale data to unit variance and zero mean.\n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/transform/scale/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/nextflow_schema.json diff --git a/target/executable/dataflow/split_h5mu/setup_logger.py b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/setup_logger.py similarity index 100% rename from target/executable/dataflow/split_h5mu/setup_logger.py rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/transform/scale/setup_logger.py diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml index be29f88..a26f556 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "gdo_singlesample" namespace: "workflows/gdo" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -225,14 +225,14 @@ build_info: output: "target/nextflow/workflows/gdo/gdo_singlesample" executable: "target/nextflow/workflows/gdo/gdo_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/filter/filter_with_counts" - "target/nextflow/filter/do_filter" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -262,7 +262,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/main.nf index f8b4b24..ebe11ad 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/main.nf @@ -1,4 +1,4 @@ -// gdo_singlesample v4.0.2 +// gdo_singlesample v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "gdo_singlesample", "namespace" : "workflows/gdo", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3327,12 +3327,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/gdo/gdo_singlesample", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3357,7 +3357,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow.config index 5979bd8..0941b1e 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/gdo/gdo_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Processing unimodal single-sample guide-derived oligonucleotide (GDO) data.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/gdo/gdo_singlesample/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml index 42fb821..86b2c65 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "dimensionality_reduction" namespace: "workflows/multiomics" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -278,14 +278,14 @@ build_info: output: "target/nextflow/workflows/multiomics/dimensionality_reduction" executable: "target/nextflow/workflows/multiomics/dimensionality_reduction/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/dimred/pca" - "target/nextflow/workflows/multiomics/neighbors_leiden_umap" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -315,7 +315,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/main.nf index 64de2ad..a9d10a1 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/main.nf @@ -1,4 +1,4 @@ -// dimensionality_reduction v4.0.2 +// dimensionality_reduction v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "dimensionality_reduction", "namespace" : "workflows/multiomics", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3390,12 +3390,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/dimensionality_reduction", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3420,7 +3420,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config index 1929ddf..236a52e 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/dimensionality_reduction' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Run calculations that output information required for most integration methods: PCA, nearest neighbour and UMAP.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/dimensionality_reduction/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml index 14ea936..ce88e7d 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "neighbors_leiden_umap" namespace: "workflows/multiomics" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -234,7 +234,7 @@ build_info: output: "target/nextflow/workflows/multiomics/neighbors_leiden_umap" executable: "target/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/cluster/leiden" @@ -243,7 +243,7 @@ build_info: - "target/nextflow/metadata/move_obsm_to_obs" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -273,7 +273,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf index 35b69c9..d9d51b7 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf @@ -1,4 +1,4 @@ -// neighbors_leiden_umap v4.0.2 +// neighbors_leiden_umap v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "neighbors_leiden_umap", "namespace" : "workflows/multiomics", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3331,12 +3331,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/neighbors_leiden_umap", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3361,7 +3361,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config index 56a0a28..8ea3fc4 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/neighbors_leiden_umap' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Performs neighborhood search, leiden clustering and run umap on an integrated embedding.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml index 9108f09..1ed92f9 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "process_batches" namespace: "workflows/multiomics" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -380,7 +380,7 @@ build_info: output: "target/nextflow/workflows/multiomics/process_batches" executable: "target/nextflow/workflows/multiomics/process_batches/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/dataflow/merge" @@ -392,7 +392,7 @@ build_info: - "target/nextflow/workflows/multiomics/dimensionality_reduction" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -422,7 +422,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/main.nf index fc200e3..c1fc3cf 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/main.nf @@ -1,4 +1,4 @@ -// process_batches v4.0.2 +// process_batches v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "process_batches", "namespace" : "workflows/multiomics", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3513,12 +3513,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/process_batches", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3543,7 +3543,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow.config index 8d44892..fdc1f13 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/process_batches' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'This workflow serves as an entrypoint into the \'full_pipeline\' in order to\nre-run the multisample processing and the integration setup. An input .h5mu file will \nfirst be split in order to run the multisample processing per modality. Next, the modalities\nare merged again and the integration setup pipeline is executed. Please note that this workflow\nassumes that samples from multiple pipelines are already concatenated. \n' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_batches/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml index ab6c2eb..342273c 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "process_samples" namespace: "workflows/multiomics" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -711,7 +711,7 @@ build_info: output: "target/nextflow/workflows/multiomics/process_samples" executable: "target/nextflow/workflows/multiomics/process_samples/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/metadata/add_id" @@ -724,7 +724,7 @@ build_info: - "target/nextflow/workflows/multiomics/process_batches" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -754,7 +754,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/main.nf index 962f560..90b44f8 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/main.nf @@ -1,4 +1,4 @@ -// process_samples v4.0.2 +// process_samples v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "process_samples", "namespace" : "workflows/multiomics", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3916,12 +3916,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/process_samples", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3946,7 +3946,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow.config index e3c88de..ce3b7f0 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/multiomics/process_samples' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'A pipeline to analyse multiple multiomics samples.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml index 088a5de..70b8351 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "prot_multisample" namespace: "workflows/prot" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -284,14 +284,14 @@ build_info: output: "target/nextflow/workflows/prot/prot_multisample" executable: "target/nextflow/workflows/prot/prot_multisample/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/transform/clr" - "target/nextflow/workflows/qc/qc" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -321,7 +321,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/main.nf index 4e5988f..e8420d7 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/main.nf @@ -1,4 +1,4 @@ -// prot_multisample v4.0.2 +// prot_multisample v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "prot_multisample", "namespace" : "workflows/prot", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3384,12 +3384,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/prot/prot_multisample", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3414,7 +3414,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow.config index f8e9137..a3414a9 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/prot/prot_multisample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Processing unimodal multi-sample ADT data.' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_multisample/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml index 72504dd..8a408b2 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "prot_singlesample" namespace: "workflows/prot" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -255,14 +255,14 @@ build_info: output: "target/nextflow/workflows/prot/prot_singlesample" executable: "target/nextflow/workflows/prot/prot_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/filter/filter_with_counts" - "target/nextflow/filter/do_filter" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -292,7 +292,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/main.nf index dce3871..f18fd6e 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/main.nf @@ -1,4 +1,4 @@ -// prot_singlesample v4.0.2 +// prot_singlesample v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "prot_singlesample", "namespace" : "workflows/prot", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3378,12 +3378,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/prot/prot_singlesample", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3408,7 +3408,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow.config index 5935c8b..fd3e866 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/prot/prot_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Processing unimodal single-sample CITE-seq data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/prot/prot_singlesample/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/.config.vsh.yaml index 8f82591..9cebe4b 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "qc" namespace: "workflows/qc" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries Schaumont" roles: @@ -364,14 +364,14 @@ build_info: output: "target/nextflow/workflows/qc/qc" executable: "target/nextflow/workflows/qc/qc/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/metadata/grep_annotation_column" - "target/nextflow/qc/calculate_qc_metrics" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -401,7 +401,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/main.nf index 6b72517..6df8306 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/main.nf @@ -1,4 +1,4 @@ -// qc v4.0.2 +// qc v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3035,7 +3035,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "qc", "namespace" : "workflows/qc", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries Schaumont", @@ -3471,12 +3471,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/qc/qc", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3501,7 +3501,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow.config index c7c3565..5659ff2 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/qc/qc' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'A pipeline to add basic qc statistics to a MuData ' author = 'Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml index 62236dd..090a424 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "rna_multisample" namespace: "workflows/rna" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -422,7 +422,7 @@ build_info: output: "target/nextflow/workflows/rna/rna_multisample" executable: "target/nextflow/workflows/rna/rna_multisample/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/feature_annotation/highly_variable_features_scanpy" @@ -432,7 +432,7 @@ build_info: - "target/_private/nextflow/workflows/rna/log_normalize" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -462,7 +462,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/main.nf index cfbe225..4918a02 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/main.nf @@ -1,4 +1,4 @@ -// rna_multisample v4.0.2 +// rna_multisample v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "rna_multisample", "namespace" : "workflows/rna", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3557,12 +3557,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/rna/rna_multisample", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3587,7 +3587,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow.config index 3656e52..4ea7c61 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/rna/rna_multisample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Processing unimodal multi-sample RNA transcriptomics data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_multisample/utils/labels_ci.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml similarity index 98% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml index f6bbb00..63e73e2 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/.config.vsh.yaml @@ -1,6 +1,6 @@ name: "rna_singlesample" namespace: "workflows/rna" -version: "v4.0.2" +version: "v4.0.3" authors: - name: "Dries De Maeyer" roles: @@ -396,7 +396,7 @@ build_info: output: "target/nextflow/workflows/rna/rna_singlesample" executable: "target/nextflow/workflows/rna/rna_singlesample/main.nf" viash_version: "0.9.4" - git_commit: "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62" + git_commit: "7bfad4ea12f87eca59213be3ab08deff67cc4206" git_remote: "https://github.com/openpipelines-bio/openpipeline" dependencies: - "target/nextflow/filter/filter_with_counts" @@ -406,7 +406,7 @@ build_info: - "target/nextflow/workflows/qc/qc" package_config: name: "openpipeline" - version: "v4.0.2" + version: "v4.0.3" summary: "Best-practice workflows for single-cell multi-omics analyses.\n" description: "OpenPipelines are extensible single cell analysis pipelines for reproducible\ \ and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\ @@ -436,7 +436,7 @@ package_config: )'" - ".engines += { type: \"native\" }" - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" - - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + - ".engines[.type == 'docker'].target_tag := 'v4.0.3'" keywords: - "single-cell" - "multimodal" diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/main.nf b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/main.nf similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/main.nf rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/main.nf index 11770df..cde5ca1 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/main.nf +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/main.nf @@ -1,4 +1,4 @@ -// rna_singlesample v4.0.2 +// rna_singlesample v4.0.3 // // This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative // work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -3037,7 +3037,7 @@ meta = [ "config": processConfig(readJsonBlob('''{ "name" : "rna_singlesample", "namespace" : "workflows/rna", - "version" : "v4.0.2", + "version" : "v4.0.3", "authors" : [ { "name" : "Dries De Maeyer", @@ -3541,12 +3541,12 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/rna/rna_singlesample", "viash_version" : "0.9.4", - "git_commit" : "cd7beddfa85b60b543399a9e2554b8e7b8eaaf62", + "git_commit" : "7bfad4ea12f87eca59213be3ab08deff67cc4206", "git_remote" : "https://github.com/openpipelines-bio/openpipeline" }, "package_config" : { "name" : "openpipeline", - "version" : "v4.0.2", + "version" : "v4.0.3", "summary" : "Best-practice workflows for single-cell multi-omics analyses.\n", "description" : "OpenPipelines are extensible single cell analysis pipelines for reproducible and large-scale single cell processing using [Viash](https://viash.io) and [Nextflow](https://www.nextflow.io/).\n\nIn terms of workflows, the following has been made available, but keep in mind that\nindividual tools and functionality can be executed as standalone components as well.\n\n * Demultiplexing: conversion of raw sequencing data to FASTQ objects.\n * Ingestion: Read mapping and generating a count matrix.\n * Single sample processing: cell filtering and doublet detection.\n * Multisample processing: Count transformation, normalization, QC metric calulations.\n * Integration: Clustering, integration and batch correction using single and multimodal methods.\n * Downstream analysis workflows\n", "info" : { @@ -3571,7 +3571,7 @@ meta = [ ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", ".engines += { type: \\"native\\" }", ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", - ".engines[.type == 'docker'].target_tag := 'v4.0.2'" + ".engines[.type == 'docker'].target_tag := 'v4.0.3'" ], "keywords" : [ "single-cell", diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow.config similarity index 99% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow.config index 42a2b9b..150305c 100644 --- a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow.config +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow.config @@ -2,7 +2,7 @@ manifest { name = 'workflows/rna/rna_singlesample' mainScript = 'main.nf' nextflowVersion = '!>=20.12.1-edge' - version = 'v4.0.2' + version = 'v4.0.3' description = 'Processing unimodal single-sample RNA transcriptomics data.' author = 'Dries De Maeyer, Robrecht Cannoodt, Dries Schaumont' } diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/labels.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow_labels.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/nextflow_schema.json diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/errorstrat_ignore.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/integration_tests.config diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels.config new file mode 100644 index 0000000..94570ec --- /dev/null +++ b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels.config @@ -0,0 +1,48 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + + // The memory a task is assinged increases with each attempt + // uncomment the line below and adjust the value to set a global upper limit on the memory. + // resourceLimits = [ memory: 240.Gb ] + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 4.GB * task.attempt } } + withLabel: midmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 25.GB * task.attempt } } + withLabel: highmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 50.GB * task.attempt } } + withLabel: veryhighmem { memory = { task?.resourceLimits?.memory && task?.maxRetries && task.attempt >= task.maxRetries ? task.resourceLimits.memory : 75.GB * task.attempt } } + + // Disk space + withLabel: lowdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: middisk { + disk = {process.disk ? process.disk : null} + } + withLabel: highdisk { + disk = {process.disk ? process.disk : null} + } + withLabel: veryhighdisk { + disk = {process.disk ? process.disk : null} + } + + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} diff --git a/target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config b/target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config similarity index 100% rename from target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config rename to target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/rna/rna_singlesample/utils/labels_ci.config diff --git a/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml index 509d1b4..77afabd 100644 --- a/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml @@ -179,7 +179,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -303,7 +303,7 @@ build_info: output: "target/executable/convert/from_cells2stats_to_h5mu" executable: "target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -317,7 +317,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu b/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu index 92e129c..936f601 100755 --- a/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu +++ b/target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu @@ -459,9 +459,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_cells2stats_to_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:58Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml index 6ab9a5d..d49d55d 100644 --- a/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml @@ -107,7 +107,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -239,7 +239,7 @@ build_info: output: "target/executable/convert/from_cosmx_to_h5mu" executable: "target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -253,7 +253,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu b/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu index 418edf7..c73c063 100755 --- a/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu +++ b/target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu @@ -460,9 +460,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_cosmx_to_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:58Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml index bf3c862..be832e6 100644 --- a/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml @@ -125,7 +125,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -234,7 +234,7 @@ build_info: output: "target/executable/convert/from_cosmx_to_spatialexperiment" executable: "target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -248,7 +248,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment b/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment index 6539685..69fdeae 100755 --- a/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment +++ b/target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment @@ -457,9 +457,9 @@ RUN Rscript -e 'options(warn = 2); if (!requireNamespace("BiocManager", quietly LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_cosmx_to_spatialexperiment" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/dataflow/split_h5mu/.config.vsh.yaml b/target/executable/convert/from_h5mu_to_spatialdata/.config.vsh.yaml similarity index 75% rename from target/executable/dataflow/split_h5mu/.config.vsh.yaml rename to target/executable/convert/from_h5mu_to_spatialdata/.config.vsh.yaml index 4feb8d4..a933b1f 100644 --- a/target/executable/dataflow/split_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_h5mu_to_spatialdata/.config.vsh.yaml @@ -1,10 +1,9 @@ -name: "split_h5mu" -namespace: "dataflow" +name: "from_h5mu_to_spatialdata" +namespace: "convert" version: "niche-compass" authors: - name: "Dorien Roosen" roles: - - "author" - "maintainer" info: role: "Core Team Member" @@ -16,22 +15,54 @@ authors: - name: "Data Intuitive" href: "https://www.data-intuitive.com" role: "Data Scientist" +- name: "Luke Zappia" + roles: + - "author" + info: + role: "Contributor" + links: + email: "luke@data-intuitive.com" + github: "lazappi" + orcid: "0000-0001-7744-8565" + linkedin: "lazappi" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Data Science Engineer" argument_groups: -- name: "Input & specifications" +- name: "Arguments" arguments: - type: "file" name: "--input" - description: "Path to a single .h5mu file." + alternatives: + - "-i" + description: "Input H5MU file." info: null + example: + - "input.h5mu" must_exist: true create_parent: true required: true direction: "input" multiple: false multiple_sep: ";" + - type: "file" + name: "--input_spatialdata" + description: "An optional existing SpatialData Zarr store to fill remaining slots\ + \ from." + info: null + example: + - "existing.zarr" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" - type: "string" name: "--modality" - description: "Which modality from the input MuData file to process.\n" + description: "The modality in the MuData to be used as the main table in the SpatialData\ + \ object." info: null default: - "rna" @@ -39,69 +70,20 @@ argument_groups: direction: "input" multiple: false multiple_sep: ";" - - type: "string" - name: "--obs_feature" - description: "The .obs column to split the mudata on." - info: null - example: - - "celltype" - required: true - direction: "input" - multiple: false - multiple_sep: ";" - - type: "boolean_true" - name: "--drop_obs_nan" - description: "Whether to drop all .obs columns that contain only nan values after\ - \ splitting." - info: null - direction: "input" - - type: "boolean_true" - name: "--ensure_unique_filenames" - description: "Append number suffixes to ensure unique filenames after sanitizing\ - \ obs feature values." - info: null - direction: "input" -- name: "Outputs" - arguments: - type: "file" name: "--output" - description: "Output directory containing multiple h5mu files." + alternatives: + - "-o" + description: "The path to the output SpatialData Zarr store." info: null example: - - "/path/to/output" + - "output.zarr" must_exist: true create_parent: true required: true direction: "output" multiple: false multiple_sep: ";" - - type: "file" - name: "--output_files" - description: "A csv containing the base filename and obs feature by which it was\ - \ split." - info: null - example: - - "sample_files.csv" - must_exist: true - create_parent: true - required: true - direction: "output" - multiple: false - multiple_sep: ";" - - type: "string" - name: "--output_compression" - description: "Compression format to use for the output AnnData and/or Mudata objects.\n\ - By default no compression is applied.\n" - info: null - example: - - "gzip" - required: false - choices: - - "gzip" - - "lzf" - direction: "input" - multiple: false - multiple_sep: ";" resources: - type: "python_script" path: "script.py" @@ -111,13 +93,18 @@ resources: - type: "file" path: "nextflow_labels.config" dest: "nextflow_labels.config" -description: "Split the samples of a single modality from a .h5mu (multimodal) sample\ - \ into seperate .h5mu files based on the values of an .obs column of this modality.\ - \ \n" +description: "Reads in an H5MU file and saves it as a SpatialData Zarr store. The\ + \ selected\nmodality in the MuData is stored as the main table in the SpatialData\ + \ object.\nIf a matching existing SpatialData is provided, it will be used to fill\ + \ the\nremaining SpatialData slots.\n \n" test_resources: - type: "python_script" path: "test.py" is_executable: true +- type: "file" + path: "xenium_tiny.h5mu" +- type: "file" + path: "xenium_tiny.zarr" info: null status: "enabled" scope: @@ -127,7 +114,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -139,9 +126,8 @@ runners: id: "nextflow" directives: label: - - "lowcpu" - - "highmem" - - "highdisk" + - "lowmem" + - "singlecpu" tag: "$id" auto: simplifyInput: true @@ -220,6 +206,9 @@ engines: - "anndata~=0.12.7" - "awkward" - "mudata~=0.3.2" + - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" + - "pyarrow~=18.0.0" script: - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ @@ -242,13 +231,13 @@ engines: - type: "native" id: "native" build_info: - config: "src/dataflow/split_h5mu/config.vsh.yaml" + config: "src/convert/from_h5mu_to_spatialdata/config.vsh.yaml" runner: "executable" engine: "docker|native" - output: "target/executable/dataflow/split_h5mu" - executable: "target/executable/dataflow/split_h5mu/split_h5mu" + output: "target/executable/convert/from_h5mu_to_spatialdata" + executable: "target/executable/convert/from_h5mu_to_spatialdata/from_h5mu_to_spatialdata" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -262,7 +251,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_h5mu_to_spatialdata/from_h5mu_to_spatialdata b/target/executable/convert/from_h5mu_to_spatialdata/from_h5mu_to_spatialdata new file mode 100755 index 0000000..a4bb5e6 --- /dev/null +++ b/target/executable/convert/from_h5mu_to_spatialdata/from_h5mu_to_spatialdata @@ -0,0 +1,1234 @@ +#!/usr/bin/env bash + +# from_h5mu_to_spatialdata niche-compass +# +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +# Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. +# +# Component authors: +# * Dorien Roosen (maintainer) +# * Luke Zappia (author) + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="from_h5mu_to_spatialdata" +VIASH_META_FUNCTIONALITY_NAME="from_h5mu_to_spatialdata" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM python:3.12-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y procps && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --upgrade pip && \ + pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" "spatialdata~=0.7.2" "ome-zarr~=0.12.2" "pyarrow~=18.0.0" && \ + python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' + +LABEL org.opencontainers.image.authors="Dorien Roosen, Luke Zappia" +LABEL org.opencontainers.image.description="Companion container for running component convert from_h5mu_to_spatialdata" +LABEL org.opencontainers.image.created="2026-03-25T10:11:20Z" +LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" +LABEL org.opencontainers.image.version="niche-compass" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "from_h5mu_to_spatialdata niche-compass" + echo "" + echo "Reads in an H5MU file and saves it as a SpatialData Zarr store. The selected" + echo "modality in the MuData is stored as the main table in the SpatialData object." + echo "If a matching existing SpatialData is provided, it will be used to fill the" + echo "remaining SpatialData slots." + echo "" + echo "Arguments:" + echo " -i, --input" + echo " type: file, required parameter, file must exist" + echo " example: input.h5mu" + echo " Input H5MU file." + echo "" + echo " --input_spatialdata" + echo " type: file, file must exist" + echo " example: existing.zarr" + echo " An optional existing SpatialData Zarr store to fill remaining slots" + echo " from." + echo "" + echo " --modality" + echo " type: string" + echo " default: rna" + echo " The modality in the MuData to be used as the main table in the" + echo " SpatialData object." + echo "" + echo " -o, --output" + echo " type: file, required parameter, output, file must exist" + echo " example: output.zarr" + echo " The path to the output SpatialData Zarr store." + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native." + echo " Default: docker" +} + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "from_h5mu_to_spatialdata niche-compass" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -i) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'-i\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -i. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input_spatialdata) + [ -n "$VIASH_PAR_INPUT_SPATIALDATA" ] && ViashError Bad arguments for option \'--input_spatialdata\': \'$VIASH_PAR_INPUT_SPATIALDATA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_SPATIALDATA="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input_spatialdata. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input_spatialdata=*) + [ -n "$VIASH_PAR_INPUT_SPATIALDATA" ] && ViashError Bad arguments for option \'--input_spatialdata=*\': \'$VIASH_PAR_INPUT_SPATIALDATA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_SPATIALDATA=$(ViashRemoveFlags "$1") + shift 1 + ;; + --modality) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --modality. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --modality=*) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality=*\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY=$(ViashRemoveFlags "$1") + shift 1 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + -o) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'-o\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to -o. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/convert/from_h5mu_to_spatialdata:niche-compass' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_MODALITY+x} ]; then + VIASH_PAR_MODALITY="rna" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi +if [ ! -z "$VIASH_PAR_INPUT_SPATIALDATA" ] && [ ! -e "$VIASH_PAR_INPUT_SPATIALDATA" ]; then + ViashError "Input file '$VIASH_PAR_INPUT_SPATIALDATA' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_INPUT_SPATIALDATA" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT_SPATIALDATA")" ) + VIASH_PAR_INPUT_SPATIALDATA=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT_SPATIALDATA") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-from_h5mu_to_spatialdata-XXXXXX").py +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +import logging +import sys + +import mudata as mu +import spatialdata as sd + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'input_spatialdata': $( if [ ! -z ${VIASH_PAR_INPUT_SPATIALDATA+x} ]; then echo "r'${VIASH_PAR_INPUT_SPATIALDATA//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\'/\'\"\'\"r\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + +sys.path.append(meta["resources_dir"]) +from setup_logger import setup_logger + +logger = setup_logger() + +logger.info("Starting conversion from H5MU to SpatialData...") + +logger.info(f"Reading input H5MU file from {par['input']}...") +mdata = mu.read_h5mu(par["input"]) + +logger.info("Extracting modality from MuData object...") +mod = mdata.mod[par["modality"]] + +if par.get("input_spatialdata", None) is not None: + logger.info(f"Reading existing SpatialData from {par['input_spatialdata']}...") + + # Disable logger messages from spatialdata when reading + logger.setLevel(logging.WARNING) + sdata_existing = sd.read_zarr(par["input_spatialdata"]) + logger.setLevel(logging.INFO) + + logger.info("Checking modality matches existing SpatialData table...") + if not mod.n_obs == sdata_existing["table"].n_obs: + raise ValueError( + "The number of observations in the selected modality does not match the existing SpatialData table." + ) + if not mod.obs_names.equals(sdata_existing["table"].obs_names): + raise ValueError( + "The observation names in the selected modality do not match the existing SpatialData table." + ) + +logger.info("Creating SpatialData object...") +if par.get("input_spatialdata", None) is not None: + logger.info("Using existing SpatialData...") + sdata = sdata_existing + sdata["table"] = mod +else: + logger.info("Creating new SpatialData...") + sdata = sd.SpatialData(tables={"table": mod}) + +logger.info(f"Writing output SpatialData Zarr store to {par['output']}...") +sdata.write(par["output"], overwrite=True) + +logger.info("Done!") +VIASHMAIN +python -B "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_INPUT_SPATIALDATA" ]; then + VIASH_PAR_INPUT_SPATIALDATA=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT_SPATIALDATA") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/dataflow/split_h5mu/nextflow_labels.config b/target/executable/convert/from_h5mu_to_spatialdata/nextflow_labels.config similarity index 100% rename from target/executable/dataflow/split_h5mu/nextflow_labels.config rename to target/executable/convert/from_h5mu_to_spatialdata/nextflow_labels.config diff --git a/target/nextflow/dataflow/split_h5mu/setup_logger.py b/target/executable/convert/from_h5mu_to_spatialdata/setup_logger.py similarity index 100% rename from target/nextflow/dataflow/split_h5mu/setup_logger.py rename to target/executable/convert/from_h5mu_to_spatialdata/setup_logger.py diff --git a/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml index 2eafc20..90f0a5d 100644 --- a/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml @@ -92,7 +92,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -198,6 +198,7 @@ engines: - "python3-pip" - "python3-dev" - "python-is-python3" + - "cmake" interactive: false - type: "r" cran: @@ -227,7 +228,7 @@ build_info: output: "target/executable/convert/from_h5mu_to_spatialexperiment" executable: "target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -241,7 +242,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment b/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment index 45e4517..ee499cf 100755 --- a/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment +++ b/target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment @@ -458,9 +458,9 @@ RUN Rscript -e 'options(warn = 2); if (!requireNamespace("remotes", quietly = TR LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_h5mu_to_spatialexperiment" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:23Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_spaceranger_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_spaceranger_to_h5mu/.config.vsh.yaml index 92a90b3..dd717a9 100644 --- a/target/executable/convert/from_spaceranger_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_spaceranger_to_h5mu/.config.vsh.yaml @@ -144,7 +144,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -265,7 +265,7 @@ build_info: output: "target/executable/convert/from_spaceranger_to_h5mu" executable: "target/executable/convert/from_spaceranger_to_h5mu/from_spaceranger_to_h5mu" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -279,7 +279,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_spaceranger_to_h5mu/from_spaceranger_to_h5mu b/target/executable/convert/from_spaceranger_to_h5mu/from_spaceranger_to_h5mu index 36a8fe0..f8f2c3d 100755 --- a/target/executable/convert/from_spaceranger_to_h5mu/from_spaceranger_to_h5mu +++ b/target/executable/convert/from_spaceranger_to_h5mu/from_spaceranger_to_h5mu @@ -458,9 +458,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_spaceranger_to_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:22Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml index db1e741..57d3b05 100644 --- a/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml @@ -101,7 +101,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -194,6 +194,7 @@ engines: - "awkward" - "mudata~=0.3.2" - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" - "pyarrow~=18.0.0" script: - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ @@ -223,7 +224,7 @@ build_info: output: "target/executable/convert/from_spatialdata_to_h5mu" executable: "target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -237,7 +238,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu b/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu index 7d82aed..8e4635f 100755 --- a/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu +++ b/target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu @@ -454,14 +454,14 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip && \ - pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" "spatialdata~=0.7.2" "pyarrow~=18.0.0" && \ + pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" "spatialdata~=0.7.2" "ome-zarr~=0.12.2" "pyarrow~=18.0.0" && \ python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_spatialdata_to_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:23Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml b/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml index f556661..4b084f5 100644 --- a/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml @@ -121,7 +121,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -246,7 +246,7 @@ build_info: output: "target/executable/convert/from_xenium_to_h5mu" executable: "target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -260,7 +260,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu b/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu index 6614117..13fe2c2 100755 --- a/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu +++ b/target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu @@ -459,9 +459,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:22Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml b/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml index 6db3058..0f0cd6a 100644 --- a/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml @@ -201,7 +201,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -295,9 +295,14 @@ engines: packages: - "spatialdata-io~=0.5.1" - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" - "pyarrow~=18.0.0" git: - "https://codeberg.org/miurahr/zipfile-inflate64.git@v0.2" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" upgrade: true test_setup: - type: "apt" @@ -326,7 +331,7 @@ build_info: output: "target/executable/convert/from_xenium_to_spatialdata" executable: "target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -340,7 +345,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata b/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata index 16a3237..7a38804 100755 --- a/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata +++ b/target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata @@ -454,14 +454,15 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip && \ - pip install --upgrade --no-cache-dir "spatialdata-io~=0.5.1" "spatialdata~=0.7.2" "pyarrow~=18.0.0" && \ - pip install --upgrade --no-cache-dir "git+https://codeberg.org/miurahr/zipfile-inflate64.git@v0.2" + pip install --upgrade --no-cache-dir "spatialdata-io~=0.5.1" "spatialdata~=0.7.2" "ome-zarr~=0.12.2" "pyarrow~=18.0.0" && \ + pip install --upgrade --no-cache-dir "git+https://codeberg.org/miurahr/zipfile-inflate64.git@v0.2" && \ + python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' LABEL org.opencontainers.image.authors="Dorien Roosen, Weiwei Schultz" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_spatialdata" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:22Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml b/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml index da8d413..fab1cae 100644 --- a/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml +++ b/target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml @@ -115,7 +115,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -210,12 +210,6 @@ engines: - "install.packages(\"arrow\", type = \"source\")" bioc_force_install: false warnings_as_errors: true - - type: "r" - script: - - "Sys.setenv(LIBARROW_MINIMAL = \"false\"); install.packages(\"arrow\", type\ - \ = \"source\")" - bioc_force_install: false - warnings_as_errors: true - type: "r" bioc: - "SpatialExperimentIO" @@ -238,7 +232,7 @@ build_info: output: "target/executable/convert/from_xenium_to_spatialexperiment" executable: "target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -252,7 +246,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment b/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment index 185ac32..28ef0f0 100755 --- a/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment +++ b/target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment @@ -455,16 +455,14 @@ RUN apt-get update && \ ENV LIBARROW_MINIMAL=false RUN Rscript -e 'options(warn = 2); install.packages("arrow", type = "source")' -RUN Rscript -e 'options(warn = 2); Sys.setenv(LIBARROW_MINIMAL = "false"); install.packages("arrow", type = "source")' - RUN Rscript -e 'options(warn = 2); if (!requireNamespace("BiocManager", quietly = TRUE)) install.packages("BiocManager")' && \ Rscript -e 'options(warn = 2); if (!requireNamespace("SpatialExperimentIO", quietly = TRUE)) BiocManager::install("SpatialExperimentIO")' LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component convert from_xenium_to_spatialexperiment" -LABEL org.opencontainers.image.created="2026-02-17T11:00:58Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:22Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/feature_annotation/spatial_autocorr/.config.vsh.yaml b/target/executable/feature_annotation/spatial_autocorr/.config.vsh.yaml new file mode 100644 index 0000000..b3c3552 --- /dev/null +++ b/target/executable/feature_annotation/spatial_autocorr/.config.vsh.yaml @@ -0,0 +1,309 @@ +name: "spatial_autocorr" +namespace: "feature_annotation" +version: "niche-compass" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "Input h5mu file." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "Modality to use." + info: null + default: + - "rna" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--layer" + description: "Layer in the AnnData object to use.\n- If `attr` is 'X', this is\ + \ the key in `.layers` to use. If not provided, `.X` is used.\n- If `attr` is\ + \ 'obsm', this is the key in `.obsm` to use.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_genes" + description: "The features to calculate autocorrelation for. Meaning depends on\ + \ `--attr`:\n- If `attr` is 'X' (default), this must be a list of gene names\ + \ from `.var_names`. \n If not provided, it defaults to using genes in `.var['highly_variable']`\ + \ (if present), or all genes.\n This behavior can be overridden by setting\ + \ `--use_all_genes` to `true`.\n Note: You cannot pass a column name from `.var`\ + \ here.\n- If `attr` is 'obs', this must be a list of column names from `.obs`.\ + \ \n- If `attr` is 'obsm', this must be indices in `.obsm[layer]` (as strings).\n" + info: null + required: false + direction: "input" + multiple: true + multiple_sep: "," + - type: "string" + name: "--obsp_neighborhood_graph" + description: "Key in .obsp where spatial connectivities are stored." + info: null + default: + - "spatial_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--use_all_genes" + description: "Whether to use all genes even if highly variable genes are present\ + \ in .var.\nIf set to true, all genes will be used.\n" + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "string" + name: "--mode" + description: "Mode of spatial autocorrelation." + info: null + default: + - "moran" + required: false + choices: + - "moran" + - "geary" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--attr" + description: "The attribute of the AnnData object to use for calculation.\n- 'X':\ + \ Use gene expression data (default).\n- 'obs': Use cell metadata.\n- 'obsm':\ + \ Use multidimensional embeddings.\n" + info: null + default: + - "X" + required: false + choices: + - "X" + - "obs" + - "obsm" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--n_perms" + description: "Number of permutations for p-value calculation." + info: null + default: + - 100 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--use_raw" + description: "Whether to use .raw attribute of AnnData." + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output h5mu file with results in .uns." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +label: "Spatial Autocorrelation" +summary: "Calculate spatial autocorrelation for genes using Moran's I or Geary's C." +description: "Calculate spatial autocorrelation for genes using Moran's I or Geary's\ + \ C.\nThis allows identifying spatially variable genes.\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.neighbors.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "midcpu" + - "midmem" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.13-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + - "scanpy~=1.10.4" + - "scanpy~=1.10.4" + - "squidpy~=1.8.1" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "python" + user: false + packages: + - "pytest" + - "viashpy" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +- type: "native" + id: "native" +build_info: + config: "src/feature_annotation/spatial_autocorr/config.vsh.yaml" + runner: "executable" + engine: "docker|native|native" + output: "target/executable/feature_annotation/spatial_autocorr" + executable: "target/executable/feature_annotation/spatial_autocorr/spatial_autocorr" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/nextflow/dataflow/split_h5mu/nextflow_labels.config b/target/executable/feature_annotation/spatial_autocorr/nextflow_labels.config similarity index 100% rename from target/nextflow/dataflow/split_h5mu/nextflow_labels.config rename to target/executable/feature_annotation/spatial_autocorr/nextflow_labels.config diff --git a/target/executable/feature_annotation/spatial_autocorr/spatial_autocorr b/target/executable/feature_annotation/spatial_autocorr/spatial_autocorr new file mode 100755 index 0000000..c3245fd --- /dev/null +++ b/target/executable/feature_annotation/spatial_autocorr/spatial_autocorr @@ -0,0 +1,1421 @@ +#!/usr/bin/env bash + +# spatial_autocorr niche-compass +# +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +# Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="spatial_autocorr" +VIASH_META_FUNCTIONALITY_NAME="spatial_autocorr" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM python:3.13-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y procps && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --upgrade pip && \ + pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" "scanpy~=1.10.4" "scanpy~=1.10.4" "squidpy~=1.8.1" && \ + python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' + +LABEL org.opencontainers.image.description="Companion container for running component feature_annotation spatial_autocorr" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" +LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" +LABEL org.opencontainers.image.version="niche-compass" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "spatial_autocorr niche-compass" + echo "" + echo "Calculate spatial autocorrelation for genes using Moran's I or Geary's C." + echo "This allows identifying spatially variable genes." + echo "" + echo "Inputs:" + echo " --input" + echo " type: file, required parameter, file must exist" + echo " Input h5mu file." + echo "" + echo " --modality" + echo " type: string" + echo " default: rna" + echo " Modality to use." + echo "" + echo " --layer" + echo " type: string" + echo " Layer in the AnnData object to use." + echo " - If \`attr\` is 'X', this is the key in \`.layers\` to use. If not" + echo " provided, \`.X\` is used." + echo " - If \`attr\` is 'obsm', this is the key in \`.obsm\` to use." + echo "" + echo " --input_genes" + echo " type: string, multiple values allowed" + echo " The features to calculate autocorrelation for. Meaning depends on" + echo " \`--attr\`:" + echo " - If \`attr\` is 'X' (default), this must be a list of gene names from" + echo " \`.var_names\`." + echo " If not provided, it defaults to using genes in" + echo " \`.var['highly_variable']\` (if present), or all genes." + echo " This behavior can be overridden by setting \`--use_all_genes\` to" + echo " \`true\`." + echo " Note: You cannot pass a column name from \`.var\` here." + echo " - If \`attr\` is 'obs', this must be a list of column names from \`.obs\`." + echo " - If \`attr\` is 'obsm', this must be indices in \`.obsm[layer]\` (as" + echo " strings)." + echo "" + echo " --obsp_neighborhood_graph" + echo " type: string" + echo " default: spatial_connectivities" + echo " Key in .obsp where spatial connectivities are stored." + echo "" + echo " --use_all_genes" + echo " type: boolean" + echo " default: false" + echo " Whether to use all genes even if highly variable genes are present in" + echo " .var." + echo " If set to true, all genes will be used." + echo "" + echo "Parameters:" + echo " --mode" + echo " type: string" + echo " default: moran" + echo " choices: [ moran, geary ]" + echo " Mode of spatial autocorrelation." + echo "" + echo " --attr" + echo " type: string" + echo " default: X" + echo " choices: [ X, obs, obsm ]" + echo " The attribute of the AnnData object to use for calculation." + echo " - 'X': Use gene expression data (default)." + echo " - 'obs': Use cell metadata." + echo " - 'obsm': Use multidimensional embeddings." + echo "" + echo " --n_perms" + echo " type: integer" + echo " default: 100" + echo " Number of permutations for p-value calculation." + echo "" + echo " --use_raw" + echo " type: boolean" + echo " default: false" + echo " Whether to use .raw attribute of AnnData." + echo "" + echo "Outputs:" + echo " --output" + echo " type: file, required parameter, output, file must exist" + echo " Output h5mu file with results in .uns." + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native, native." + echo " Default: docker" +} + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "spatial_autocorr niche-compass" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --modality) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --modality. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --modality=*) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality=*\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY=$(ViashRemoveFlags "$1") + shift 1 + ;; + --layer) + [ -n "$VIASH_PAR_LAYER" ] && ViashError Bad arguments for option \'--layer\': \'$VIASH_PAR_LAYER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_LAYER="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --layer. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --layer=*) + [ -n "$VIASH_PAR_LAYER" ] && ViashError Bad arguments for option \'--layer=*\': \'$VIASH_PAR_LAYER\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_LAYER=$(ViashRemoveFlags "$1") + shift 1 + ;; + --input_genes) + if [ -z "$VIASH_PAR_INPUT_GENES" ]; then + VIASH_PAR_INPUT_GENES="$2" + else + VIASH_PAR_INPUT_GENES="$VIASH_PAR_INPUT_GENES,""$2" + fi + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input_genes. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input_genes=*) + if [ -z "$VIASH_PAR_INPUT_GENES" ]; then + VIASH_PAR_INPUT_GENES=$(ViashRemoveFlags "$1") + else + VIASH_PAR_INPUT_GENES="$VIASH_PAR_INPUT_GENES,"$(ViashRemoveFlags "$1") + fi + shift 1 + ;; + --obsp_neighborhood_graph) + [ -n "$VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH" ] && ViashError Bad arguments for option \'--obsp_neighborhood_graph\': \'$VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --obsp_neighborhood_graph. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --obsp_neighborhood_graph=*) + [ -n "$VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH" ] && ViashError Bad arguments for option \'--obsp_neighborhood_graph=*\': \'$VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH=$(ViashRemoveFlags "$1") + shift 1 + ;; + --use_all_genes) + [ -n "$VIASH_PAR_USE_ALL_GENES" ] && ViashError Bad arguments for option \'--use_all_genes\': \'$VIASH_PAR_USE_ALL_GENES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_USE_ALL_GENES="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --use_all_genes. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --use_all_genes=*) + [ -n "$VIASH_PAR_USE_ALL_GENES" ] && ViashError Bad arguments for option \'--use_all_genes=*\': \'$VIASH_PAR_USE_ALL_GENES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_USE_ALL_GENES=$(ViashRemoveFlags "$1") + shift 1 + ;; + --mode) + [ -n "$VIASH_PAR_MODE" ] && ViashError Bad arguments for option \'--mode\': \'$VIASH_PAR_MODE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODE="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --mode. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --mode=*) + [ -n "$VIASH_PAR_MODE" ] && ViashError Bad arguments for option \'--mode=*\': \'$VIASH_PAR_MODE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODE=$(ViashRemoveFlags "$1") + shift 1 + ;; + --attr) + [ -n "$VIASH_PAR_ATTR" ] && ViashError Bad arguments for option \'--attr\': \'$VIASH_PAR_ATTR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ATTR="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --attr. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --attr=*) + [ -n "$VIASH_PAR_ATTR" ] && ViashError Bad arguments for option \'--attr=*\': \'$VIASH_PAR_ATTR\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ATTR=$(ViashRemoveFlags "$1") + shift 1 + ;; + --n_perms) + [ -n "$VIASH_PAR_N_PERMS" ] && ViashError Bad arguments for option \'--n_perms\': \'$VIASH_PAR_N_PERMS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_N_PERMS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --n_perms. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --n_perms=*) + [ -n "$VIASH_PAR_N_PERMS" ] && ViashError Bad arguments for option \'--n_perms=*\': \'$VIASH_PAR_N_PERMS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_N_PERMS=$(ViashRemoveFlags "$1") + shift 1 + ;; + --use_raw) + [ -n "$VIASH_PAR_USE_RAW" ] && ViashError Bad arguments for option \'--use_raw\': \'$VIASH_PAR_USE_RAW\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_USE_RAW="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --use_raw. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --use_raw=*) + [ -n "$VIASH_PAR_USE_RAW" ] && ViashError Bad arguments for option \'--use_raw=*\': \'$VIASH_PAR_USE_RAW\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_USE_RAW=$(ViashRemoveFlags "$1") + shift 1 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/feature_annotation/spatial_autocorr:niche-compass' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_MODALITY+x} ]; then + VIASH_PAR_MODALITY="rna" +fi +if [ -z ${VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH+x} ]; then + VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH="spatial_connectivities" +fi +if [ -z ${VIASH_PAR_USE_ALL_GENES+x} ]; then + VIASH_PAR_USE_ALL_GENES="false" +fi +if [ -z ${VIASH_PAR_MODE+x} ]; then + VIASH_PAR_MODE="moran" +fi +if [ -z ${VIASH_PAR_ATTR+x} ]; then + VIASH_PAR_ATTR="X" +fi +if [ -z ${VIASH_PAR_N_PERMS+x} ]; then + VIASH_PAR_N_PERMS="100" +fi +if [ -z ${VIASH_PAR_USE_RAW+x} ]; then + VIASH_PAR_USE_RAW="false" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_PAR_USE_ALL_GENES" ]]; then + if ! [[ "$VIASH_PAR_USE_ALL_GENES" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--use_all_genes' has to be a boolean. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_N_PERMS" ]]; then + if ! [[ "$VIASH_PAR_N_PERMS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--n_perms' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_USE_RAW" ]]; then + if ! [[ "$VIASH_PAR_USE_RAW" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--use_raw' has to be a boolean. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# check whether value is belongs to a set of choices +if [ ! -z "$VIASH_PAR_MODE" ]; then + VIASH_PAR_MODE_CHOICES=("moran;geary") + IFS=';' + set -f + if ! [[ ";${VIASH_PAR_MODE_CHOICES[*]};" =~ ";$VIASH_PAR_MODE;" ]]; then + ViashError '--mode' specified value of \'$VIASH_PAR_MODE\' is not in the list of allowed values. Use "--help" to get more information on the parameters. + exit 1 + fi + set +f + unset IFS +fi + +if [ ! -z "$VIASH_PAR_ATTR" ]; then + VIASH_PAR_ATTR_CHOICES=("X;obs;obsm") + IFS=';' + set -f + if ! [[ ";${VIASH_PAR_ATTR_CHOICES[*]};" =~ ";$VIASH_PAR_ATTR;" ]]; then + ViashError '--attr' specified value of \'$VIASH_PAR_ATTR\' is not in the list of allowed values. Use "--help" to get more information on the parameters. + exit 1 + fi + set +f + unset IFS +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-spatial_autocorr-XXXXXX").py +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +import mudata as mu +import squidpy as sq + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'layer': $( if [ ! -z ${VIASH_PAR_LAYER+x} ]; then echo "r'${VIASH_PAR_LAYER//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'input_genes': $( if [ ! -z ${VIASH_PAR_INPUT_GENES+x} ]; then echo "r'${VIASH_PAR_INPUT_GENES//\'/\'\"\'\"r\'}'.split(',')"; else echo None; fi ), + 'obsp_neighborhood_graph': $( if [ ! -z ${VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH+x} ]; then echo "r'${VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'use_all_genes': $( if [ ! -z ${VIASH_PAR_USE_ALL_GENES+x} ]; then echo "r'${VIASH_PAR_USE_ALL_GENES//\'/\'\"\'\"r\'}'.lower() == 'true'"; else echo None; fi ), + 'mode': $( if [ ! -z ${VIASH_PAR_MODE+x} ]; then echo "r'${VIASH_PAR_MODE//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'attr': $( if [ ! -z ${VIASH_PAR_ATTR+x} ]; then echo "r'${VIASH_PAR_ATTR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'n_perms': $( if [ ! -z ${VIASH_PAR_N_PERMS+x} ]; then echo "int(r'${VIASH_PAR_N_PERMS//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'use_raw': $( if [ ! -z ${VIASH_PAR_USE_RAW+x} ]; then echo "r'${VIASH_PAR_USE_RAW//\'/\'\"\'\"r\'}'.lower() == 'true'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\'/\'\"\'\"r\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + + +def main(): + print("Reading input MuData...", flush=True) + mdata = mu.read_h5mu(par["input"]) + adata = mdata.mod[par["modality"]] + + # Check for connectivity key + if par["obsp_neighborhood_graph"] not in adata.obsp: + raise ValueError( + f"Connectivity key '{par['obsp_neighborhood_graph']}' not found in .obsp of modality '{par['modality']}'." + ) + + genes = par["input_genes"] + if genes and len(genes) == 0: + genes = None + + if genes is None and par["use_all_genes"] and par["attr"] == "X": + genes = list(adata.var_names) + + # Handle layer + layer = par["layer"] + if layer == "None": + layer = None + + print(f"Calculating spatial autocorrelation ({par['mode']})...", flush=True) + + # Run Squidpy spatial_autocorr + # Note: sq.gr.spatial_autocorr modifies adata in-place, adding results to .uns + sq.gr.spatial_autocorr( + adata, + connectivity_key=par["obsp_neighborhood_graph"], + genes=genes, + mode=par["mode"], + attr=par["attr"], + n_perms=par["n_perms"], + layer=layer, + use_raw=par["use_raw"], + ) + + result_key = f"{par['mode']}I" if par["mode"] == "moran" else f"{par['mode']}C" + + if result_key in adata.uns: + # Log top spatially variable genes + df = adata.uns[result_key] + if not df.empty: + sort_col = "I" if par["mode"] == "moran" else "C" + print("Top spatially variable genes:", flush=True) + print(df.sort_values(by=sort_col, ascending=False).head(), flush=True) + else: + print( + f"Warning: Expected key '{result_key}' not found in .uns after calculation.", + flush=True, + ) + + print("Writing output...", flush=True) + mdata.write_h5mu(par["output"]) + print("Done!", flush=True) + + +if __name__ == "__main__": + main() +VIASHMAIN +python -B "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml b/target/executable/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml new file mode 100644 index 0000000..0192970 --- /dev/null +++ b/target/executable/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml @@ -0,0 +1,279 @@ +name: "xenium_spatial_statistics" +namespace: "feature_annotation" +version: "niche-compass" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "A MuData file containing spatial transcriptomics data and a\npre-computed\ + \ spatial neighborhood graph in `.obsp`.\n" + info: null + example: + - "input.h5mu" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "The name of the modality to use from the MuData object.\nThis specifies\ + \ which AnnData object contains the spatial data.\n" + info: null + default: + - "rna" + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--obsm_spatial_coordinates" + description: "The key in `.obsm` where spatial coordinates are stored.\nExpected\ + \ shape is (n_cells, 2) for 2D coordinates.\n" + info: null + default: + - "spatial" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "The output MuData file with calculated spatial statistics.\nPer-cell\ + \ metrics are stored in `.obs` and global statistics in `.uns`.\n" + info: null + default: + - "output.h5mu" + must_exist: true + create_parent: true + required: false + direction: "output" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_prefix" + description: "Prefix to add to all generated column names in `.obs`.\n" + info: null + default: + - "spatial_" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "double" + name: "--density_bandwidth" + description: "Bandwidth parameter for Gaussian kernel density estimation in spatial\ + \ units\n(typically microns). Larger values create smoother density estimates.\n" + info: null + default: + - 50.0 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--calculate_ripley_l" + description: "Whether to calculate Ripley's L statistic. \nWarning: This is an\ + \ O(N^2) operation and can be very slow for large datasets (>5000 cells).\n" + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--n_subsample_ripley" + description: "Number of cells to subsample for Ripley's L calculation. \nIf -1,\ + \ use all cells. Recommended to keep this below 5000 for performance.\n" + info: null + default: + - -1 + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +label: "Xenium Spatial Statistics" +description: "Calculate various spatial statistics from Xenium data including cell\ + \ morphology\nratios, position-based features, local density metrics, and global\ + \ spatial patterns.\nThis component expects a MuData object with a pre-computed\ + \ spatial neighborhood graph\nin `.obsp`.\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.neighbors.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "singlecpu" + - "midmem" + - "lowdisk" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.13-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + - "scanpy~=1.10.4" + - "scanpy~=1.10.4" + - "squidpy~=1.8.1" + - "scikit-learn" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "python" + user: false + packages: + - "pytest" + - "viashpy" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +- type: "native" + id: "native" +build_info: + config: "src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml" + runner: "executable" + engine: "docker|native|native" + output: "target/executable/feature_annotation/xenium_spatial_statistics" + executable: "target/executable/feature_annotation/xenium_spatial_statistics/xenium_spatial_statistics" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/executable/feature_annotation/xenium_spatial_statistics/nextflow_labels.config b/target/executable/feature_annotation/xenium_spatial_statistics/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/executable/feature_annotation/xenium_spatial_statistics/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/executable/feature_annotation/xenium_spatial_statistics/xenium_spatial_statistics b/target/executable/feature_annotation/xenium_spatial_statistics/xenium_spatial_statistics new file mode 100755 index 0000000..52eb566 --- /dev/null +++ b/target/executable/feature_annotation/xenium_spatial_statistics/xenium_spatial_statistics @@ -0,0 +1,1571 @@ +#!/usr/bin/env bash + +# xenium_spatial_statistics niche-compass +# +# This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +# Intuitive. +# +# The component may contain files which fall under a different license. The +# authors of this component should specify the license in the header of such +# files, or include a separate license file detailing the licenses of all included +# files. + +set -e + +if [ -z "$VIASH_TEMP" ]; then + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$VIASH_TMP} + VIASH_TEMP=${VIASH_TEMP:-$TMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TMP} + VIASH_TEMP=${VIASH_TEMP:-$TEMPDIR} + VIASH_TEMP=${VIASH_TEMP:-$TEMP} + VIASH_TEMP=${VIASH_TEMP:-/tmp} +fi + +# define helper functions +# ViashQuote: put quotes around non flag values +# $1 : unquoted string +# return : possibly quoted string +# examples: +# ViashQuote --foo # returns --foo +# ViashQuote bar # returns 'bar' +# Viashquote --foo=bar # returns --foo='bar' +function ViashQuote { + if [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+=.+$ ]]; then + echo "$1" | sed "s#=\(.*\)#='\1'#" + elif [[ "$1" =~ ^-+[a-zA-Z0-9_\-]+$ ]]; then + echo "$1" + else + echo "'$1'" + fi +} +# ViashRemoveFlags: Remove leading flag +# $1 : string with a possible leading flag +# return : string without possible leading flag +# examples: +# ViashRemoveFlags --foo=bar # returns bar +function ViashRemoveFlags { + echo "$1" | sed 's/^--*[a-zA-Z0-9_\-]*=//' +} +# ViashSourceDir: return the path of a bash file, following symlinks +# usage : ViashSourceDir ${BASH_SOURCE[0]} +# $1 : Should always be set to ${BASH_SOURCE[0]} +# returns : The absolute path of the bash file +function ViashSourceDir { + local source="$1" + while [ -h "$source" ]; do + local dir="$( cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd )" + source="$(readlink "$source")" + [[ $source != /* ]] && source="$dir/$source" + done + cd -P "$( dirname "$source" )" >/dev/null 2>&1 && pwd +} +# ViashFindTargetDir: return the path of the '.build.yaml' file, following symlinks +# usage : ViashFindTargetDir 'ScriptPath' +# $1 : The location from where to start the upward search +# returns : The absolute path of the '.build.yaml' file +function ViashFindTargetDir { + local source="$1" + while [[ "$source" != "" && ! -e "$source/.build.yaml" ]]; do + source=${source%/*} + done + echo $source +} +# see https://en.wikipedia.org/wiki/Syslog#Severity_level +VIASH_LOGCODE_EMERGENCY=0 +VIASH_LOGCODE_ALERT=1 +VIASH_LOGCODE_CRITICAL=2 +VIASH_LOGCODE_ERROR=3 +VIASH_LOGCODE_WARNING=4 +VIASH_LOGCODE_NOTICE=5 +VIASH_LOGCODE_INFO=6 +VIASH_LOGCODE_DEBUG=7 +VIASH_VERBOSITY=$VIASH_LOGCODE_NOTICE + +# ViashLog: Log events depending on the verbosity level +# usage: ViashLog 1 alert Oh no something went wrong! +# $1: required verbosity level +# $2: display tag +# $3+: messages to display +# stdout: Your input, prepended by '[$2] '. +function ViashLog { + local required_level="$1" + local display_tag="$2" + shift 2 + if [ $VIASH_VERBOSITY -ge $required_level ]; then + >&2 echo "[$display_tag]" "$@" + fi +} + +# ViashEmergency: log events when the system is unstable +# usage: ViashEmergency Oh no something went wrong. +# stdout: Your input, prepended by '[emergency] '. +function ViashEmergency { + ViashLog $VIASH_LOGCODE_EMERGENCY emergency "$@" +} + +# ViashAlert: log events when actions must be taken immediately (e.g. corrupted system database) +# usage: ViashAlert Oh no something went wrong. +# stdout: Your input, prepended by '[alert] '. +function ViashAlert { + ViashLog $VIASH_LOGCODE_ALERT alert "$@" +} + +# ViashCritical: log events when a critical condition occurs +# usage: ViashCritical Oh no something went wrong. +# stdout: Your input, prepended by '[critical] '. +function ViashCritical { + ViashLog $VIASH_LOGCODE_CRITICAL critical "$@" +} + +# ViashError: log events when an error condition occurs +# usage: ViashError Oh no something went wrong. +# stdout: Your input, prepended by '[error] '. +function ViashError { + ViashLog $VIASH_LOGCODE_ERROR error "$@" +} + +# ViashWarning: log potentially abnormal events +# usage: ViashWarning Something may have gone wrong. +# stdout: Your input, prepended by '[warning] '. +function ViashWarning { + ViashLog $VIASH_LOGCODE_WARNING warning "$@" +} + +# ViashNotice: log significant but normal events +# usage: ViashNotice This just happened. +# stdout: Your input, prepended by '[notice] '. +function ViashNotice { + ViashLog $VIASH_LOGCODE_NOTICE notice "$@" +} + +# ViashInfo: log normal events +# usage: ViashInfo This just happened. +# stdout: Your input, prepended by '[info] '. +function ViashInfo { + ViashLog $VIASH_LOGCODE_INFO info "$@" +} + +# ViashDebug: log all events, for debugging purposes +# usage: ViashDebug This just happened. +# stdout: Your input, prepended by '[debug] '. +function ViashDebug { + ViashLog $VIASH_LOGCODE_DEBUG debug "$@" +} + +# find source folder of this component +VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` + +# find the root of the built components & dependencies +VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` + +# define meta fields +VIASH_META_NAME="xenium_spatial_statistics" +VIASH_META_FUNCTIONALITY_NAME="xenium_spatial_statistics" +VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" +VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" +VIASH_META_TEMP_DIR="$VIASH_TEMP" + + + +# initialise variables +VIASH_MODE='run' +VIASH_ENGINE_ID='docker' + +######## Helper functions for setting up Docker images for viash ######## +# expects: ViashDockerBuild + +# ViashDockerInstallationCheck: check whether Docker is installed correctly +# +# examples: +# ViashDockerInstallationCheck +function ViashDockerInstallationCheck { + ViashDebug "Checking whether Docker is installed" + if [ ! command -v docker &> /dev/null ]; then + ViashCritical "Docker doesn't seem to be installed. See 'https://docs.docker.com/get-docker/' for instructions." + exit 1 + fi + + ViashDebug "Checking whether the Docker daemon is running" + local save=$-; set +e + local docker_version=$(docker version --format '{{.Client.APIVersion}}' 2> /dev/null) + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashCritical "Docker daemon does not seem to be running. Try one of the following:" + ViashCritical "- Try running 'dockerd' in the command line" + ViashCritical "- See https://docs.docker.com/config/daemon/" + exit 1 + fi +} + +# ViashDockerRemoteTagCheck: check whether a Docker image is available +# on a remote. Assumes `docker login` has been performed, if relevant. +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerRemoteTagCheck python:latest +# echo $? # returns '0' +# ViashDockerRemoteTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerRemoteTagCheck { + docker manifest inspect $1 > /dev/null 2> /dev/null +} + +# ViashDockerLocalTagCheck: check whether a Docker image is available locally +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# docker pull python:latest +# ViashDockerLocalTagCheck python:latest +# echo $? # returns '0' +# ViashDockerLocalTagCheck sdaizudceahifu +# echo $? # returns '1' +function ViashDockerLocalTagCheck { + [ -n "$(docker images -q $1)" ] +} + +# ViashDockerPull: pull a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPull python:latest +# echo $? # returns '0' +# ViashDockerPull sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPull { + ViashNotice "Checking if Docker image is available at '$1'" + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker pull $1 && return 0 || return 1 + else + local save=$-; set +e + docker pull $1 2> /dev/null > /dev/null + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashWarning "Could not pull from '$1'. Docker image doesn't exist or is not accessible." + fi + return $out + fi +} + +# ViashDockerPush: push a Docker image +# +# $1 : image identifier with format `[registry/]image[:tag]` +# exit code $? : whether or not the image was found +# examples: +# ViashDockerPush python:latest +# echo $? # returns '0' +# ViashDockerPush sdaizudceahifu +# echo $? # returns '1' +function ViashDockerPush { + ViashNotice "Pushing image to '$1'" + local save=$-; set +e + local out + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + docker push $1 + out=$? + else + docker push $1 2> /dev/null > /dev/null + out=$? + fi + [[ $save =~ e ]] && set -e + if [ $out -eq 0 ]; then + ViashNotice "Container '$1' push succeeded." + else + ViashError "Container '$1' push errored. You might not be logged in or have the necessary permissions." + fi + return $out +} + +# ViashDockerPullElseBuild: pull a Docker image, else build it +# +# $1 : image identifier with format `[registry/]image[:tag]` +# ViashDockerBuild : a Bash function which builds a docker image, takes image identifier as argument. +# examples: +# ViashDockerPullElseBuild mynewcomponent +function ViashDockerPullElseBuild { + local save=$-; set +e + ViashDockerPull $1 + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashDockerBuild $@ + fi +} + +# ViashDockerSetup: create a Docker image, according to specified docker setup strategy +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $2 : docker setup strategy, see DockerSetupStrategy.scala +# examples: +# ViashDockerSetup mynewcomponent alwaysbuild +function ViashDockerSetup { + local image_id="$1" + local setup_strategy="$2" + if [ "$setup_strategy" == "alwaysbuild" -o "$setup_strategy" == "build" -o "$setup_strategy" == "b" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspull" -o "$setup_strategy" == "pull" -o "$setup_strategy" == "p" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "alwayspullelsebuild" -o "$setup_strategy" == "pullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayspullelsecachedbuild" -o "$setup_strategy" == "pullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "alwayscachedbuild" -o "$setup_strategy" == "cachedbuild" -o "$setup_strategy" == "cb" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [[ "$setup_strategy" =~ ^ifneedbe ]]; then + local save=$-; set +e + ViashDockerLocalTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashInfo "Image $image_id already exists" + elif [ "$setup_strategy" == "ifneedbebuild" ]; then + ViashDockerBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbecachedbuild" ]; then + ViashDockerBuild $image_id $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepull" ]; then + ViashDockerPull $image_id + elif [ "$setup_strategy" == "ifneedbepullelsebuild" ]; then + ViashDockerPullElseBuild $image_id --no-cache $(ViashDockerBuildArgs "$engine_id") + elif [ "$setup_strategy" == "ifneedbepullelsecachedbuild" ]; then + ViashDockerPullElseBuild $image_id $(ViashDockerBuildArgs "$engine_id") + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi + elif [ "$setup_strategy" == "push" -o "$setup_strategy" == "forcepush" -o "$setup_strategy" == "alwayspush" ]; then + ViashDockerPush "$image_id" + elif [ "$setup_strategy" == "pushifnotpresent" -o "$setup_strategy" == "gentlepush" -o "$setup_strategy" == "maybepush" ]; then + local save=$-; set +e + ViashDockerRemoteTagCheck $image_id + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -eq 0 ]; then + ViashNotice "Container '$image_id' exists, doing nothing." + else + ViashNotice "Container '$image_id' does not yet exist." + ViashDockerPush "$image_id" + fi + elif [ "$setup_strategy" == "donothing" -o "$setup_strategy" == "meh" ]; then + ViashNotice "Skipping setup." + else + ViashError "Unrecognised Docker strategy: $setup_strategy" + exit 1 + fi +} + +# ViashDockerCheckCommands: Check whether a docker container has the required commands +# +# $1 : image identifier with format `[registry/]image[:tag]` +# $@ : commands to verify being present +# examples: +# ViashDockerCheckCommands bash:4.0 bash ps foo +function ViashDockerCheckCommands { + local image_id="$1" + shift 1 + local commands="$@" + local save=$-; set +e + local missing # mark 'missing' as local in advance, otherwise the exit code of the command will be missing and always be '0' + missing=$(docker run --rm --entrypoint=sh "$image_id" -c "for command in $commands; do command -v \$command >/dev/null 2>&1; if [ \$? -ne 0 ]; then echo \$command; exit 1; fi; done") + local outCheck=$? + [[ $save =~ e ]] && set -e + if [ $outCheck -ne 0 ]; then + ViashError "Docker container '$image_id' does not contain command '$missing'." + exit 1 + fi +} + +# ViashDockerBuild: build a docker image +# $1 : image identifier with format `[registry/]image[:tag]` +# $... : additional arguments to pass to docker build +# $VIASH_META_TEMP_DIR : temporary directory to store dockerfile & optional resources in +# $VIASH_META_NAME : name of the component +# $VIASH_META_RESOURCES_DIR : directory containing the resources +# $VIASH_VERBOSITY : verbosity level +# exit code $? : whether or not the image was built successfully +function ViashDockerBuild { + local image_id="$1" + shift 1 + + # create temporary directory to store dockerfile & optional resources in + local tmpdir=$(mktemp -d "$VIASH_META_TEMP_DIR/dockerbuild-$VIASH_META_NAME-XXXXXX") + local dockerfile="$tmpdir/Dockerfile" + function clean_up { + rm -rf "$tmpdir" + } + trap clean_up EXIT + + # store dockerfile and resources + ViashDockerfile "$VIASH_ENGINE_ID" > "$dockerfile" + + # generate the build command + local docker_build_cmd="docker build -t '$image_id' $@ '$VIASH_META_RESOURCES_DIR' -f '$dockerfile'" + + # build the container + ViashNotice "Building container '$image_id' with Dockerfile" + ViashInfo "$docker_build_cmd" + local save=$-; set +e + if [ $VIASH_VERBOSITY -ge $VIASH_LOGCODE_INFO ]; then + eval $docker_build_cmd + else + eval $docker_build_cmd &> "$tmpdir/docker_build.log" + fi + + # check exit code + local out=$? + [[ $save =~ e ]] && set -e + if [ $out -ne 0 ]; then + ViashError "Error occurred while building container '$image_id'" + if [ $VIASH_VERBOSITY -lt $VIASH_LOGCODE_INFO ]; then + ViashError "Transcript: --------------------------------" + cat "$tmpdir/docker_build.log" + ViashError "End of transcript --------------------------" + fi + exit 1 + fi +} + +######## End of helper functions for setting up Docker images for viash ######## + +# ViashDockerFile: print the dockerfile to stdout +# $1 : engine identifier +# return : dockerfile required to run this component +# examples: +# ViashDockerFile +function ViashDockerfile { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + cat << 'VIASHDOCKER' +FROM python:3.13-slim +ENTRYPOINT [] +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y procps && \ + rm -rf /var/lib/apt/lists/* + +RUN pip install --upgrade pip && \ + pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" "scanpy~=1.10.4" "scanpy~=1.10.4" "squidpy~=1.8.1" "scikit-learn" && \ + python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' + +LABEL org.opencontainers.image.description="Companion container for running component feature_annotation xenium_spatial_statistics" +LABEL org.opencontainers.image.created="2026-03-25T10:11:22Z" +LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" +LABEL org.opencontainers.image.version="niche-compass" + +VIASHDOCKER + fi +} + +# ViashDockerBuildArgs: return the arguments to pass to docker build +# $1 : engine identifier +# return : arguments to pass to docker build +function ViashDockerBuildArgs { + local engine_id="$1" + + if [[ "$engine_id" == "docker" ]]; then + echo "" + fi +} + +# ViashAbsolutePath: generate absolute path from relative path +# borrowed from https://stackoverflow.com/a/21951256 +# $1 : relative filename +# return : absolute path +# examples: +# ViashAbsolutePath some_file.txt # returns /path/to/some_file.txt +# ViashAbsolutePath /foo/bar/.. # returns /foo +function ViashAbsolutePath { + local thePath + local parr + local outp + local len + if [[ ! "$1" =~ ^/ ]]; then + thePath="$PWD/$1" + else + thePath="$1" + fi + echo "$thePath" | ( + IFS=/ + read -a parr + declare -a outp + for i in "${parr[@]}"; do + case "$i" in + ''|.) continue ;; + ..) + len=${#outp[@]} + if ((len==0)); then + continue + else + unset outp[$((len-1))] + fi + ;; + *) + len=${#outp[@]} + outp[$len]="$i" + ;; + esac + done + echo /"${outp[*]}" + ) +} +# ViashDockerAutodetectMount: auto configuring docker mounts from parameters +# $1 : The parameter value +# returns : New parameter +# $VIASH_DIRECTORY_MOUNTS : Added another parameter to be passed to docker +# $VIASH_DOCKER_AUTOMOUNT_PREFIX : The prefix to be used for the automounts +# examples: +# ViashDockerAutodetectMount /path/to/bar # returns '/viash_automount/path/to/bar' +# ViashDockerAutodetectMountArg /path/to/bar # returns '--volume="/path/to:/viash_automount/path/to"' +function ViashDockerAutodetectMount { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + if [ -z "$base_name" ]; then + echo "$mount_target" + else + echo "$mount_target/$base_name" + fi +} +function ViashDockerAutodetectMountArg { + local abs_path=$(ViashAbsolutePath "$1") + local mount_source + local base_name + if [ -d "$abs_path" ]; then + mount_source="$abs_path" + base_name="" + else + mount_source=`dirname "$abs_path"` + base_name=`basename "$abs_path"` + fi + local mount_target="$VIASH_DOCKER_AUTOMOUNT_PREFIX$mount_source" + ViashDebug "ViashDockerAutodetectMountArg $1 -> $mount_source -> $mount_target" + echo "--volume=\"$mount_source:$mount_target\"" +} +function ViashDockerStripAutomount { + local abs_path=$(ViashAbsolutePath "$1") + echo "${abs_path#$VIASH_DOCKER_AUTOMOUNT_PREFIX}" +} +# initialise variables +VIASH_DIRECTORY_MOUNTS=() + +# configure default docker automount prefix if it is unset +if [ -z "${VIASH_DOCKER_AUTOMOUNT_PREFIX+x}" ]; then + VIASH_DOCKER_AUTOMOUNT_PREFIX="/viash_automount" +fi + +# initialise docker variables +VIASH_DOCKER_RUN_ARGS=(-i --rm) + + +# ViashHelp: Display helpful explanation about this executable +function ViashHelp { + echo "xenium_spatial_statistics niche-compass" + echo "" + echo "Calculate various spatial statistics from Xenium data including cell morphology" + echo "ratios, position-based features, local density metrics, and global spatial" + echo "patterns." + echo "This component expects a MuData object with a pre-computed spatial neighborhood" + echo "graph" + echo "in \`.obsp\`." + echo "" + echo "Inputs:" + echo " --input" + echo " type: file, required parameter, file must exist" + echo " example: input.h5mu" + echo " A MuData file containing spatial transcriptomics data and a" + echo " pre-computed spatial neighborhood graph in \`.obsp\`." + echo "" + echo " --modality" + echo " type: string, required parameter" + echo " default: rna" + echo " The name of the modality to use from the MuData object." + echo " This specifies which AnnData object contains the spatial data." + echo "" + echo " --obsm_spatial_coordinates" + echo " type: string" + echo " default: spatial" + echo " The key in \`.obsm\` where spatial coordinates are stored." + echo " Expected shape is (n_cells, 2) for 2D coordinates." + echo "" + echo "Outputs:" + echo " --output" + echo " type: file, output, file must exist" + echo " default: output.h5mu" + echo " The output MuData file with calculated spatial statistics." + echo " Per-cell metrics are stored in \`.obs\` and global statistics in \`.uns\`." + echo "" + echo " --output_prefix" + echo " type: string" + echo " default: spatial_" + echo " Prefix to add to all generated column names in \`.obs\`." + echo "" + echo "Parameters:" + echo " --density_bandwidth" + echo " type: double" + echo " default: 50.0" + echo " Bandwidth parameter for Gaussian kernel density estimation in spatial" + echo " units" + echo " (typically microns). Larger values create smoother density estimates." + echo "" + echo " --calculate_ripley_l" + echo " type: boolean" + echo " default: false" + echo " Whether to calculate Ripley's L statistic." + echo " Warning: This is an O(N^2) operation and can be very slow for large" + echo " datasets (>5000 cells)." + echo "" + echo " --n_subsample_ripley" + echo " type: integer" + echo " default: -1" + echo " Number of cells to subsample for Ripley's L calculation." + echo " If -1, use all cells. Recommended to keep this below 5000 for" + echo " performance." + echo "" + echo "Viash built in Computational Requirements:" + echo " ---cpus=INT" + echo " Number of CPUs to use" + echo " ---memory=STRING" + echo " Amount of memory to use. Examples: 4GB, 3MiB." + echo "" + echo "Viash built in Docker:" + echo " ---setup=STRATEGY" + echo " Setup the docker container. Options are: alwaysbuild, alwayscachedbuild, ifneedbebuild, ifneedbecachedbuild, alwayspull, alwayspullelsebuild, alwayspullelsecachedbuild, ifneedbepull, ifneedbepullelsebuild, ifneedbepullelsecachedbuild, push, pushifnotpresent, donothing." + echo " Default: ifneedbepullelsecachedbuild" + echo " ---dockerfile" + echo " Print the dockerfile to stdout." + echo " ---docker_run_args=ARG" + echo " Provide runtime arguments to Docker. See the documentation on \`docker run\` for more information." + echo " ---docker_image_id" + echo " Print the docker image id to stdout." + echo " ---debug" + echo " Enter the docker container for debugging purposes." + echo "" + echo "Viash built in Engines:" + echo " ---engine=ENGINE_ID" + echo " Specify the engine to use. Options are: docker, native, native." + echo " Default: docker" +} + +# initialise array +VIASH_POSITIONAL_ARGS='' + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + ViashHelp + exit + ;; + ---v|---verbose) + let "VIASH_VERBOSITY=VIASH_VERBOSITY+1" + shift 1 + ;; + ---verbosity) + VIASH_VERBOSITY="$2" + shift 2 + ;; + ---verbosity=*) + VIASH_VERBOSITY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + --version) + echo "xenium_spatial_statistics niche-compass" + exit + ;; + --input) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input=*) + [ -n "$VIASH_PAR_INPUT" ] && ViashError Bad arguments for option \'--input=*\': \'$VIASH_PAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --modality) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --modality. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --modality=*) + [ -n "$VIASH_PAR_MODALITY" ] && ViashError Bad arguments for option \'--modality=*\': \'$VIASH_PAR_MODALITY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_MODALITY=$(ViashRemoveFlags "$1") + shift 1 + ;; + --obsm_spatial_coordinates) + [ -n "$VIASH_PAR_OBSM_SPATIAL_COORDINATES" ] && ViashError Bad arguments for option \'--obsm_spatial_coordinates\': \'$VIASH_PAR_OBSM_SPATIAL_COORDINATES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OBSM_SPATIAL_COORDINATES="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --obsm_spatial_coordinates. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --obsm_spatial_coordinates=*) + [ -n "$VIASH_PAR_OBSM_SPATIAL_COORDINATES" ] && ViashError Bad arguments for option \'--obsm_spatial_coordinates=*\': \'$VIASH_PAR_OBSM_SPATIAL_COORDINATES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OBSM_SPATIAL_COORDINATES=$(ViashRemoveFlags "$1") + shift 1 + ;; + --output) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output=*) + [ -n "$VIASH_PAR_OUTPUT" ] && ViashError Bad arguments for option \'--output=*\': \'$VIASH_PAR_OUTPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; + --output_prefix) + [ -n "$VIASH_PAR_OUTPUT_PREFIX" ] && ViashError Bad arguments for option \'--output_prefix\': \'$VIASH_PAR_OUTPUT_PREFIX\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT_PREFIX="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output_prefix. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --output_prefix=*) + [ -n "$VIASH_PAR_OUTPUT_PREFIX" ] && ViashError Bad arguments for option \'--output_prefix=*\': \'$VIASH_PAR_OUTPUT_PREFIX\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT_PREFIX=$(ViashRemoveFlags "$1") + shift 1 + ;; + --density_bandwidth) + [ -n "$VIASH_PAR_DENSITY_BANDWIDTH" ] && ViashError Bad arguments for option \'--density_bandwidth\': \'$VIASH_PAR_DENSITY_BANDWIDTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DENSITY_BANDWIDTH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --density_bandwidth. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --density_bandwidth=*) + [ -n "$VIASH_PAR_DENSITY_BANDWIDTH" ] && ViashError Bad arguments for option \'--density_bandwidth=*\': \'$VIASH_PAR_DENSITY_BANDWIDTH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_DENSITY_BANDWIDTH=$(ViashRemoveFlags "$1") + shift 1 + ;; + --calculate_ripley_l) + [ -n "$VIASH_PAR_CALCULATE_RIPLEY_L" ] && ViashError Bad arguments for option \'--calculate_ripley_l\': \'$VIASH_PAR_CALCULATE_RIPLEY_L\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_CALCULATE_RIPLEY_L="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --calculate_ripley_l. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --calculate_ripley_l=*) + [ -n "$VIASH_PAR_CALCULATE_RIPLEY_L" ] && ViashError Bad arguments for option \'--calculate_ripley_l=*\': \'$VIASH_PAR_CALCULATE_RIPLEY_L\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_CALCULATE_RIPLEY_L=$(ViashRemoveFlags "$1") + shift 1 + ;; + --n_subsample_ripley) + [ -n "$VIASH_PAR_N_SUBSAMPLE_RIPLEY" ] && ViashError Bad arguments for option \'--n_subsample_ripley\': \'$VIASH_PAR_N_SUBSAMPLE_RIPLEY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_N_SUBSAMPLE_RIPLEY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --n_subsample_ripley. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --n_subsample_ripley=*) + [ -n "$VIASH_PAR_N_SUBSAMPLE_RIPLEY" ] && ViashError Bad arguments for option \'--n_subsample_ripley=*\': \'$VIASH_PAR_N_SUBSAMPLE_RIPLEY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_N_SUBSAMPLE_RIPLEY=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---engine) + VIASH_ENGINE_ID="$2" + shift 2 + ;; + ---engine=*) + VIASH_ENGINE_ID="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---setup) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$2" + shift 2 + ;; + ---setup=*) + VIASH_MODE='setup' + VIASH_SETUP_STRATEGY="$(ViashRemoveFlags "$1")" + shift 1 + ;; + ---dockerfile) + VIASH_MODE='dockerfile' + shift 1 + ;; + ---docker_run_args) + VIASH_DOCKER_RUN_ARGS+=("$2") + shift 2 + ;; + ---docker_run_args=*) + VIASH_DOCKER_RUN_ARGS+=("$(ViashRemoveFlags "$1")") + shift 1 + ;; + ---docker_image_id) + VIASH_MODE='docker_image_id' + shift 1 + ;; + ---debug) + VIASH_MODE='debug' + shift 1 + ;; + ---cpus) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---cpus. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---cpus=*) + [ -n "$VIASH_META_CPUS" ] && ViashError Bad arguments for option \'---cpus=*\': \'$VIASH_META_CPUS\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_CPUS=$(ViashRemoveFlags "$1") + shift 1 + ;; + ---memory) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to ---memory. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + ---memory=*) + [ -n "$VIASH_META_MEMORY" ] && ViashError Bad arguments for option \'---memory=*\': \'$VIASH_META_MEMORY\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_META_MEMORY=$(ViashRemoveFlags "$1") + shift 1 + ;; + *) # positional arg or unknown option + # since the positional args will be eval'd, can we always quote, instead of using ViashQuote + VIASH_POSITIONAL_ARGS="$VIASH_POSITIONAL_ARGS '$1'" + [[ $1 == -* ]] && ViashWarning $1 looks like a parameter but is not a defined parameter and will instead be treated as a positional argument. Use "--help" to get more information on the parameters. + shift # past argument + ;; + esac +done + +# parse positional parameters +eval set -- $VIASH_POSITIONAL_ARGS + + +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then + VIASH_ENGINE_TYPE='native' +elif [ "$VIASH_ENGINE_ID" == "docker" ] ; then + VIASH_ENGINE_TYPE='docker' +else + ViashError "Engine '$VIASH_ENGINE_ID' is not recognized. Options are: docker, native, native." + exit 1 +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # check if docker is installed properly + ViashDockerInstallationCheck + + # determine docker image id + if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/feature_annotation/xenium_spatial_statistics:niche-compass' + fi + + # print dockerfile + if [ "$VIASH_MODE" == "dockerfile" ]; then + ViashDockerfile "$VIASH_ENGINE_ID" + exit 0 + + elif [ "$VIASH_MODE" == "docker_image_id" ]; then + echo "$VIASH_DOCKER_IMAGE_ID" + exit 0 + + # enter docker container + elif [[ "$VIASH_MODE" == "debug" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} -v '$(pwd)':/pwd --workdir /pwd -t $VIASH_DOCKER_IMAGE_ID" + ViashNotice "+ $VIASH_CMD" + eval $VIASH_CMD + exit + + # build docker image + elif [ "$VIASH_MODE" == "setup" ]; then + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" "$VIASH_SETUP_STRATEGY" + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' + exit 0 + fi + + # check if docker image exists + ViashDockerSetup "$VIASH_DOCKER_IMAGE_ID" ifneedbepullelsecachedbuild + ViashDockerCheckCommands "$VIASH_DOCKER_IMAGE_ID" 'bash' +fi + +# setting computational defaults + +# helper function for parsing memory strings +function ViashMemoryAsBytes { + local memory=`echo "$1" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]'` + local memory_regex='^([0-9]+)([kmgtp]i?b?|b)$' + if [[ $memory =~ $memory_regex ]]; then + local number=${memory/[^0-9]*/} + local symbol=${memory/*[0-9]/} + + case $symbol in + b) memory_b=$number ;; + kb|k) memory_b=$(( $number * 1000 )) ;; + mb|m) memory_b=$(( $number * 1000 * 1000 )) ;; + gb|g) memory_b=$(( $number * 1000 * 1000 * 1000 )) ;; + tb|t) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 )) ;; + pb|p) memory_b=$(( $number * 1000 * 1000 * 1000 * 1000 * 1000 )) ;; + kib|ki) memory_b=$(( $number * 1024 )) ;; + mib|mi) memory_b=$(( $number * 1024 * 1024 )) ;; + gib|gi) memory_b=$(( $number * 1024 * 1024 * 1024 )) ;; + tib|ti) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 )) ;; + pib|pi) memory_b=$(( $number * 1024 * 1024 * 1024 * 1024 * 1024 )) ;; + esac + echo "$memory_b" + fi +} +# compute memory in different units +if [ ! -z ${VIASH_META_MEMORY+x} ]; then + VIASH_META_MEMORY_B=`ViashMemoryAsBytes $VIASH_META_MEMORY` + # do not define other variables if memory_b is an empty string + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_META_MEMORY_KB=$(( ($VIASH_META_MEMORY_B+999) / 1000 )) + VIASH_META_MEMORY_MB=$(( ($VIASH_META_MEMORY_KB+999) / 1000 )) + VIASH_META_MEMORY_GB=$(( ($VIASH_META_MEMORY_MB+999) / 1000 )) + VIASH_META_MEMORY_TB=$(( ($VIASH_META_MEMORY_GB+999) / 1000 )) + VIASH_META_MEMORY_PB=$(( ($VIASH_META_MEMORY_TB+999) / 1000 )) + VIASH_META_MEMORY_KIB=$(( ($VIASH_META_MEMORY_B+1023) / 1024 )) + VIASH_META_MEMORY_MIB=$(( ($VIASH_META_MEMORY_KIB+1023) / 1024 )) + VIASH_META_MEMORY_GIB=$(( ($VIASH_META_MEMORY_MIB+1023) / 1024 )) + VIASH_META_MEMORY_TIB=$(( ($VIASH_META_MEMORY_GIB+1023) / 1024 )) + VIASH_META_MEMORY_PIB=$(( ($VIASH_META_MEMORY_TIB+1023) / 1024 )) + else + # unset memory if string is empty + unset $VIASH_META_MEMORY_B + fi +fi +# unset nproc if string is empty +if [ -z "$VIASH_META_CPUS" ]; then + unset $VIASH_META_CPUS +fi + + +# check whether required parameters exist +if [ -z ${VIASH_PAR_INPUT+x} ]; then + ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_PAR_MODALITY+x} ]; then + ViashError '--modality' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_NAME+x} ]; then + ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then + ViashError 'functionality_name' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_RESOURCES_DIR+x} ]; then + ViashError 'resources_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_EXECUTABLE+x} ]; then + ViashError 'executable' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_CONFIG+x} ]; then + ViashError 'config' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi +if [ -z ${VIASH_META_TEMP_DIR+x} ]; then + ViashError 'temp_dir' is a required argument. Use "--help" to get more information on the parameters. + exit 1 +fi + +# filling in defaults +if [ -z ${VIASH_PAR_OBSM_SPATIAL_COORDINATES+x} ]; then + VIASH_PAR_OBSM_SPATIAL_COORDINATES="spatial" +fi +if [ -z ${VIASH_PAR_OUTPUT+x} ]; then + VIASH_PAR_OUTPUT="output.h5mu" +fi +if [ -z ${VIASH_PAR_OUTPUT_PREFIX+x} ]; then + VIASH_PAR_OUTPUT_PREFIX="spatial_" +fi +if [ -z ${VIASH_PAR_DENSITY_BANDWIDTH+x} ]; then + VIASH_PAR_DENSITY_BANDWIDTH="50.0" +fi +if [ -z ${VIASH_PAR_CALCULATE_RIPLEY_L+x} ]; then + VIASH_PAR_CALCULATE_RIPLEY_L="false" +fi +if [ -z ${VIASH_PAR_N_SUBSAMPLE_RIPLEY+x} ]; then + VIASH_PAR_N_SUBSAMPLE_RIPLEY="-1" +fi + +# check whether required files exist +if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then + ViashError "Input file '$VIASH_PAR_INPUT' does not exist." + exit 1 +fi + +# check whether parameters values are of the right type +if [[ -n "$VIASH_PAR_DENSITY_BANDWIDTH" ]]; then + if ! [[ "$VIASH_PAR_DENSITY_BANDWIDTH" =~ ^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$ ]]; then + ViashError '--density_bandwidth' has to be a double. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_CALCULATE_RIPLEY_L" ]]; then + if ! [[ "$VIASH_PAR_CALCULATE_RIPLEY_L" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then + ViashError '--calculate_ripley_l' has to be a boolean. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_PAR_N_SUBSAMPLE_RIPLEY" ]]; then + if ! [[ "$VIASH_PAR_N_SUBSAMPLE_RIPLEY" =~ ^[-+]?[0-9]+$ ]]; then + ViashError '--n_subsample_ripley' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_CPUS" ]]; then + if ! [[ "$VIASH_META_CPUS" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'cpus' has to be an integer. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_B" ]]; then + if ! [[ "$VIASH_META_MEMORY_B" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_b' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pb' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_KIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_KIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_kib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_MIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_MIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_mib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_GIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_GIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_gib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_TIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_TIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_tib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi +if [[ -n "$VIASH_META_MEMORY_PIB" ]]; then + if ! [[ "$VIASH_META_MEMORY_PIB" =~ ^[-+]?[0-9]+$ ]]; then + ViashError 'memory_pib' has to be a long. Use "--help" to get more information on the parameters. + exit 1 + fi +fi + +# create parent directories of output files, if so desired +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then + mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" +fi + +if [ "$VIASH_ENGINE_ID" == "native" ] || [ "$VIASH_ENGINE_ID" == "native" ] ; then + if [ "$VIASH_MODE" == "run" ]; then + VIASH_CMD="bash" + else + ViashError "Engine '$VIASH_ENGINE_ID' does not support mode '$VIASH_MODE'." + exit 1 + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # detect volumes from file arguments + VIASH_CHOWN_VARS=() +if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_INPUT")" ) + VIASH_PAR_INPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_INPUT") +fi +if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT")" ) + VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") + VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) +fi +if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) + VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") +fi +if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_EXECUTABLE")" ) + VIASH_META_EXECUTABLE=$(ViashDockerAutodetectMount "$VIASH_META_EXECUTABLE") +fi +if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_CONFIG")" ) + VIASH_META_CONFIG=$(ViashDockerAutodetectMount "$VIASH_META_CONFIG") +fi +if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_TEMP_DIR")" ) + VIASH_META_TEMP_DIR=$(ViashDockerAutodetectMount "$VIASH_META_TEMP_DIR") +fi + + # get unique mounts + VIASH_UNIQUE_MOUNTS=($(for val in "${VIASH_DIRECTORY_MOUNTS[@]}"; do echo "$val"; done | sort -u)) +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # change file ownership + function ViashPerformChown { + if (( ${#VIASH_CHOWN_VARS[@]} )); then + set +e + VIASH_CMD="docker run --entrypoint=bash --rm ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID -c 'chown $(id -u):$(id -g) --silent --recursive ${VIASH_CHOWN_VARS[@]}'" + ViashDebug "+ $VIASH_CMD" + eval $VIASH_CMD + set -e + fi + } + trap ViashPerformChown EXIT +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # helper function for filling in extra docker args + if [ ! -z "$VIASH_META_MEMORY_B" ]; then + VIASH_DOCKER_RUN_ARGS+=("--memory=${VIASH_META_MEMORY_B}") + fi + if [ ! -z "$VIASH_META_CPUS" ]; then + VIASH_DOCKER_RUN_ARGS+=("--cpus=${VIASH_META_CPUS}") + fi +fi + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + VIASH_CMD="docker run --entrypoint=bash ${VIASH_DOCKER_RUN_ARGS[@]} ${VIASH_UNIQUE_MOUNTS[@]} $VIASH_DOCKER_IMAGE_ID" +fi + + +# set dependency paths + + +ViashDebug "Running command: $(echo $VIASH_CMD)" +cat << VIASHEOF | eval $VIASH_CMD +set -e +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-xenium_spatial_statistics-XXXXXX").py +function clean_up { + rm "\$tempscript" +} +function interrupt { + echo -e "\nCTRL-C Pressed..." + exit 1 +} +trap clean_up EXIT +trap interrupt INT SIGINT +cat > "\$tempscript" << 'VIASHMAIN' +import sys +import warnings + +import mudata as mu +import numpy as np +import squidpy as sq +from scipy.spatial import ConvexHull, Voronoi, distance_matrix +from sklearn.neighbors import KernelDensity + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'obsm_spatial_coordinates': $( if [ ! -z ${VIASH_PAR_OBSM_SPATIAL_COORDINATES+x} ]; then echo "r'${VIASH_PAR_OBSM_SPATIAL_COORDINATES//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'output_prefix': $( if [ ! -z ${VIASH_PAR_OUTPUT_PREFIX+x} ]; then echo "r'${VIASH_PAR_OUTPUT_PREFIX//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'density_bandwidth': $( if [ ! -z ${VIASH_PAR_DENSITY_BANDWIDTH+x} ]; then echo "float(r'${VIASH_PAR_DENSITY_BANDWIDTH//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'calculate_ripley_l': $( if [ ! -z ${VIASH_PAR_CALCULATE_RIPLEY_L+x} ]; then echo "r'${VIASH_PAR_CALCULATE_RIPLEY_L//\'/\'\"\'\"r\'}'.lower() == 'true'"; else echo None; fi ), + 'n_subsample_ripley': $( if [ ! -z ${VIASH_PAR_N_SUBSAMPLE_RIPLEY+x} ]; then echo "int(r'${VIASH_PAR_N_SUBSAMPLE_RIPLEY//\'/\'\"\'\"r\'}')"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\'/\'\"\'\"r\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\'/\'\"\'\"r\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + + +def calculate_morphology_metrics(adata, prefix): + """Calculate cell morphology ratios and features.""" + print(" Calculating morphology metrics...", flush=True) + + if "cell_area" not in adata.obs or "nucleus_area" not in adata.obs: + warnings.warn( + "cell_area and/or nucleus_area not found in .obs. " + "Skipping morphology metrics." + ) + return + + # Nucleus to cell area ratio - can be removed once https://github.com/openpipelines-bio/openpipeline_qc/issues/18 is fixed. + adata.obs[f"{prefix}nucleus_cell_ratio"] = ( + adata.obs["nucleus_area"] / adata.obs["cell_area"] + ) + + # Cell area percentile (relative size) + adata.obs[f"{prefix}cell_area_percentile"] = ( + adata.obs["cell_area"].rank(pct=True) * 100 + ) + + print( + f" Added: {prefix}nucleus_cell_ratio, {prefix}cell_area_percentile", + flush=True, + ) + + +def calculate_position_features(adata, spatial_coords, prefix): + """Calculate position-based features.""" + print(" Calculating position-based features...", flush=True) + + # Tissue centroid + centroid = spatial_coords.mean(axis=0) + + # Distance to centroid + distances_to_centroid = np.linalg.norm(spatial_coords - centroid, axis=1) + adata.obs[f"{prefix}distance_to_centroid"] = distances_to_centroid + + # Normalized coordinates (0-1 scale) + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + coord_range = max_coords - min_coords + + normalized_coords = (spatial_coords - min_coords) / coord_range + adata.obs[f"{prefix}norm_x"] = normalized_coords[:, 0] + adata.obs[f"{prefix}norm_y"] = normalized_coords[:, 1] + + # Distance to convex hull boundary + try: + hull = ConvexHull(spatial_coords) + hull_points = spatial_coords[hull.vertices] + + # For each point, find distance to nearest hull vertex (approximation) + distances_to_boundary = np.min( + distance_matrix(spatial_coords, hull_points), axis=1 + ) + adata.obs[f"{prefix}distance_to_boundary"] = distances_to_boundary + print( + " Added: distance_to_centroid, norm_x, norm_y, distance_to_boundary", + flush=True, + ) + except Exception as e: + warnings.warn(f"Could not calculate convex hull: {e}") + print(" Added: distance_to_centroid, norm_x, norm_y", flush=True) + + +def calculate_density_metrics(adata, spatial_coords, bandwidth, prefix): + """Calculate local density metrics.""" + print(" Calculating density metrics...", flush=True) + + # Kernel density estimation + kde = KernelDensity(bandwidth=bandwidth, kernel="gaussian") + kde.fit(spatial_coords) + log_density = kde.score_samples(spatial_coords) + adata.obs[f"{prefix}kernel_density"] = np.exp(log_density) + + # Calculate degree centrality (as a proxy for local density / number of neighbors) + # This assumes spatial_connectivities is already present in .obsp (from build_spatial_graph) + if "spatial_connectivities" in adata.obsp: + spatial_connectivities = adata.obsp["spatial_connectivities"] + degrees = spatial_connectivities.sum(axis=1) + # If sparse matrix, it returns matrix object, need to convert to array + if hasattr(degrees, "A1"): + degrees = degrees.A1 + adata.obs[f"{prefix}graph_degree"] = degrees + + print(" Added: kernel_density, graph_degree", flush=True) + else: + print( + " Warning: 'spatial_connectivities' not found in .obsp. Skipping graph_degree.", + flush=True, + ) + print(" Added: kernel_density", flush=True) + + +def calculate_voronoi_metrics(adata, spatial_coords, prefix): + """Calculate Voronoi tessellation statistics.""" + print(" Calculating Voronoi tessellation...", flush=True) + + try: + vor = Voronoi(spatial_coords) + + polygon_areas = [] + neighbor_counts = [] + + for point_idx in range(len(spatial_coords)): + region_idx = vor.point_region[point_idx] + region = vor.regions[region_idx] + + # Skip infinite regions + if -1 in region or len(region) == 0: + polygon_areas.append(np.nan) + neighbor_counts.append(np.nan) + continue + + # Calculate polygon area using shoelace formula + vertices = vor.vertices[region] + x = vertices[:, 0] + y = vertices[:, 1] + area = 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + polygon_areas.append(area) + + # Count neighbors (shared vertices) + neighbor_counts.append(len(region)) + + adata.obs[f"{prefix}voronoi_area"] = polygon_areas + adata.obs[f"{prefix}voronoi_neighbors"] = neighbor_counts + + print(" Added: voronoi_area, voronoi_neighbors", flush=True) + except Exception as e: + warnings.warn(f"Could not calculate Voronoi tessellation: {e}") + + +def calculate_global_statistics(adata, spatial_coords, par): + """Calculate global spatial pattern statistics.""" + print(" Calculating global spatial statistics...", flush=True) + + stats = {} + + # Global density + try: + hull = ConvexHull(spatial_coords) + area = hull.volume # In 2D, volume of convex hull is the area + stats["area_calculation_method"] = "convex_hull" + stats["cell_density"] = len(spatial_coords) / area + stats["total_area"] = area + stats["n_cells"] = len(spatial_coords) + except Exception as e: + print(f" Error: Could not calculate convex hull area ({e})", flush=True) + + # Ripley's L (using squidpy) + if par["calculate_ripley_l"]: + print(" Computing Ripley's L function (Squidpy)...", flush=True) + + n_subsample = par["n_subsample_ripley"] + + try: + # Create a working AnnData for Ripley's + if n_subsample > 0 and len(adata) > n_subsample: + print(f" Subsampling to {n_subsample} random cells...", flush=True) + + # Use numpy to generate random indices + indices = np.random.choice(len(adata), n_subsample, replace=False) + adata_ripley = adata[indices].copy() + stats["ripley_l_subsampled"] = True + else: + adata_ripley = adata.copy() + stats["ripley_l_subsampled"] = False + + # Squidpy requires a cluster key. We create a dummy one for "global" context. + adata_ripley.obs["_temp_global"] = "all" + adata_ripley.obs["_temp_global"] = adata_ripley.obs["_temp_global"].astype( + "category" + ) + + # Calculate Ripley's L statistic + # This stores the result in adata.uns['all_L'] + sq.gr.ripley( + adata_ripley, + cluster_key="_temp_global", + mode="L", + spatial_key=par["obsm_spatial_coordinates"], + n_simulations=50, + ) + + # Extract basic stats from the results + if "all_L" in adata_ripley.uns and "L_stat" in adata_ripley.uns["all_L"]: + l_stat_df = adata_ripley.uns["all_L"]["L_stat"] + stats["ripley_l_max"] = float(l_stat_df.max().max()) + stats["ripley_l_mean"] = float(l_stat_df.mean().mean()) + + # Clean up + if "all_L" in adata_ripley.uns: + del adata_ripley.uns["all_L"] + + except Exception as e: + print(f" Error calculating Ripley's L: {e}", flush=True) + import traceback + + traceback.print_exc() + else: + print(" Skipping Ripley's L (disabled in config)...", flush=True) + + # Spatial extent + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + + stats["spatial_extent_x"] = float(max_coords[0] - min_coords[0]) + stats["spatial_extent_y"] = float(max_coords[1] - min_coords[1]) + stats["centroid_x"] = float(spatial_coords[:, 0].mean()) + stats["centroid_y"] = float(spatial_coords[:, 1].mean()) + + if "cell_density" in stats: + print( + f" Global stats: cell_density={stats['cell_density']:.4f}", + flush=True, + ) + + return stats + + +def main(par): + print(f"\\n>>> Reading MuData from '{par['input']}'...", flush=True) + mdata = mu.read_h5mu(par["input"]) + print(mdata, flush=True) + + print(f"\\n>>> Extracting modality '{par['modality']}'...", flush=True) + if par["modality"] not in mdata.mod: + raise KeyError( + f"Modality '{par['modality']}' not found in MuData. " + f"Available modalities: {list(mdata.mod.keys())}" + ) + adata = mdata[par["modality"]] + print(adata, flush=True) + + print( + f"\\n>>> Extracting spatial coordinates from .obsm['{par['obsm_spatial_coordinates']}']...", + flush=True, + ) + if par["obsm_spatial_coordinates"] not in adata.obsm: + raise KeyError( + f"Spatial key '{par['obsm_spatial_coordinates']}' not found in .obsm. " + f"Available keys: {list(adata.obsm.keys())}" + ) + + spatial_coords = adata.obsm[par["obsm_spatial_coordinates"]] + if spatial_coords.shape[1] != 2: + raise ValueError( + f"Expected 2D spatial coordinates, got shape {spatial_coords.shape}" + ) + print(f" Shape: {spatial_coords.shape} (n_cells × 2)", flush=True) + + prefix = par["output_prefix"] + + # Calculate morphology metrics + print("\\n>>> Calculating morphology metrics...", flush=True) + calculate_morphology_metrics(adata, prefix) + + # Calculate position features + print("\\n>>> Calculating position-based features...", flush=True) + calculate_position_features(adata, spatial_coords, prefix) + + # Calculate density metrics + print("\\n>>> Calculating density metrics...", flush=True) + calculate_density_metrics( + adata, + spatial_coords, + par["density_bandwidth"], + prefix, + ) + + # Calculate Voronoi tessellation + print("\\n>>> Calculating Voronoi tessellation...", flush=True) + calculate_voronoi_metrics(adata, spatial_coords, prefix) + + # Calculate global statistics + print("\\n>>> Calculating global spatial statistics...", flush=True) + global_stats = calculate_global_statistics(adata, spatial_coords, par) + + # Store in uns + if "spatial_stats" not in adata.uns: + adata.uns["spatial_stats"] = {} + adata.uns["spatial_stats"].update(global_stats) + + print(f"\\n>>> Writing output to '{par['output']}'...", flush=True) + mdata.write_h5mu(par["output"]) + + print("\\n>>> Done!\\n", flush=True) + + +if __name__ == "__main__": + sys.exit(main(par)) +VIASHMAIN +python -B "\$tempscript" & +wait "\$!" + +VIASHEOF + + +if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then + # strip viash automount from file paths + + if [ ! -z "$VIASH_PAR_INPUT" ]; then + VIASH_PAR_INPUT=$(ViashDockerStripAutomount "$VIASH_PAR_INPUT") + fi + if [ ! -z "$VIASH_PAR_OUTPUT" ]; then + VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") + fi + if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then + VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") + fi + if [ ! -z "$VIASH_META_EXECUTABLE" ]; then + VIASH_META_EXECUTABLE=$(ViashDockerStripAutomount "$VIASH_META_EXECUTABLE") + fi + if [ ! -z "$VIASH_META_CONFIG" ]; then + VIASH_META_CONFIG=$(ViashDockerStripAutomount "$VIASH_META_CONFIG") + fi + if [ ! -z "$VIASH_META_TEMP_DIR" ]; then + VIASH_META_TEMP_DIR=$(ViashDockerStripAutomount "$VIASH_META_TEMP_DIR") + fi +fi + + +# check whether required files exist +if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then + ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." + exit 1 +fi + + +exit 0 diff --git a/target/executable/mapping/spaceranger_count/.config.vsh.yaml b/target/executable/mapping/spaceranger_count/.config.vsh.yaml index a69e224..58cf3da 100644 --- a/target/executable/mapping/spaceranger_count/.config.vsh.yaml +++ b/target/executable/mapping/spaceranger_count/.config.vsh.yaml @@ -330,7 +330,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" keywords: - "spaceranger" links: @@ -439,7 +439,7 @@ build_info: output: "target/executable/mapping/spaceranger_count" executable: "target/executable/mapping/spaceranger_count/spaceranger_count" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -453,7 +453,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/mapping/spaceranger_count/spaceranger_count b/target/executable/mapping/spaceranger_count/spaceranger_count index 4552170..60e2c10 100755 --- a/target/executable/mapping/spaceranger_count/spaceranger_count +++ b/target/executable/mapping/spaceranger_count/spaceranger_count @@ -453,9 +453,9 @@ apt upgrade -y && apt install -y procps && rm -rf /var/lib/apt/lists/* LABEL org.opencontainers.image.authors="Jakub Majercik" LABEL org.opencontainers.image.description="Companion container for running component mapping spaceranger_count" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:20Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/neighbors/join_graphs/.config.vsh.yaml b/target/executable/neighbors/join_graphs/.config.vsh.yaml new file mode 100644 index 0000000..c6c8dcc --- /dev/null +++ b/target/executable/neighbors/join_graphs/.config.vsh.yaml @@ -0,0 +1,295 @@ +name: "join_graphs" +namespace: "neighbors" +version: "niche-compass" +authors: +- name: "Jakub Majercik" + roles: + - "maintainer" + info: + role: "Contributor" + links: + email: "jakub@data-intuitive.com" + github: "jakubmajercik" + linkedin: "jakubmajercik" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatics Engineer" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "Input H5MU file with pre-computed expression and spatial neighborhood\ + \ graphs." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "Modality from the input MuData file to process." + info: null + default: + - "rna" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_obsp_expression_graph" + description: "Key in `adata.obsp` containing the expression connectivity matrix.\n" + info: null + default: + - "connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_obsp_spatial_graph" + description: "Key in `adata.obsp` containing the spatial connectivity matrix.\n" + info: null + default: + - "spatial_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "double" + name: "--alpha" + description: "Weight of the spatial graph in the connection fusion. \nA value\ + \ of 0 results in the original expression graph; \na value of 1 results in the\ + \ spatial graph only.\n" + info: null + default: + - 0.5 + required: false + min: 0.0 + max: 1.0 + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output H5MU file path." + info: null + example: + - "output.h5mu" + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_obsp_graph" + description: "Key under which to store the fused connectivity matrix in `adata.obsp`.\n" + info: null + default: + - "spatial_expression_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_compression" + description: "Compression format to use for the output AnnData and/or Mudata objects.\n\ + By default no compression is applied.\n" + info: null + example: + - "gzip" + required: false + choices: + - "gzip" + - "lzf" + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "setup_logger.py" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +description: "Combine spatial and expression neighborhood graphs into a single graph\ + \ for \nuse in downstream clustering or trajectory analysis.\n\nThe fusion is a\ + \ linear combination of the expression connectivities \nand spatial connectivities\ + \ matrices, weighted by `alpha`.\n\n$C_{joint} = (1 - \\alpha) \\cdot C_{expression}\ + \ + \\alpha \\cdot C_{spatial}$\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.all_neighbors.pca.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "midcpu" + - "midmem" + - "middisk" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.12-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "scanpy~=1.10.4" + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "apt" + packages: + - "git" + interactive: false + - type: "python" + user: false + packages: + - "viashpy==0.9.0" + github: + - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/neighbors/join_graphs/config.vsh.yaml" + runner: "executable" + engine: "docker|native" + output: "target/executable/neighbors/join_graphs" + executable: "target/executable/neighbors/join_graphs/join_graphs" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/executable/dataflow/split_h5mu/split_h5mu b/target/executable/neighbors/join_graphs/join_graphs similarity index 82% rename from target/executable/dataflow/split_h5mu/split_h5mu rename to target/executable/neighbors/join_graphs/join_graphs index a12a2d1..4d13f52 100755 --- a/target/executable/dataflow/split_h5mu/split_h5mu +++ b/target/executable/neighbors/join_graphs/join_graphs @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# split_h5mu niche-compass +# join_graphs niche-compass # # This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative # work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data @@ -12,7 +12,7 @@ # files. # # Component authors: -# * Dorien Roosen (author, maintainer) +# * Jakub Majercik (maintainer) set -e @@ -165,8 +165,8 @@ VIASH_META_RESOURCES_DIR=`ViashSourceDir ${BASH_SOURCE[0]}` VIASH_TARGET_DIR=`ViashFindTargetDir $VIASH_META_RESOURCES_DIR` # define meta fields -VIASH_META_NAME="split_h5mu" -VIASH_META_FUNCTIONALITY_NAME="split_h5mu" +VIASH_META_NAME="join_graphs" +VIASH_META_FUNCTIONALITY_NAME="join_graphs" VIASH_META_EXECUTABLE="$VIASH_META_RESOURCES_DIR/$VIASH_META_NAME" VIASH_META_CONFIG="$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" VIASH_META_TEMP_DIR="$VIASH_TEMP" @@ -453,14 +453,14 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* RUN pip install --upgrade pip && \ - pip install --upgrade --no-cache-dir "anndata~=0.12.7" "awkward" "mudata~=0.3.2" && \ + pip install --upgrade --no-cache-dir "scanpy~=1.10.4" "anndata~=0.12.7" "awkward" "mudata~=0.3.2" && \ python -c 'exec("try:\n import zarr; from importlib.metadata import version\nexcept ModuleNotFoundError:\n exit(0)\nelse: assert int(version(\"zarr\").partition(\".\")[0]) > 2")' -LABEL org.opencontainers.image.authors="Dorien Roosen" -LABEL org.opencontainers.image.description="Companion container for running component dataflow split_h5mu" -LABEL org.opencontainers.image.created="2026-02-17T11:00:59Z" +LABEL org.opencontainers.image.authors="Jakub Majercik" +LABEL org.opencontainers.image.description="Companion container for running component neighbors join_graphs" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER @@ -578,47 +578,57 @@ VIASH_DOCKER_RUN_ARGS=(-i --rm) # ViashHelp: Display helpful explanation about this executable function ViashHelp { - echo "split_h5mu niche-compass" + echo "join_graphs niche-compass" echo "" - echo "Split the samples of a single modality from a .h5mu (multimodal) sample into" - echo "seperate .h5mu files based on the values of an .obs column of this modality." + echo "Combine spatial and expression neighborhood graphs into a single graph for" + echo "use in downstream clustering or trajectory analysis." echo "" - echo "Input & specifications:" + echo "The fusion is a linear combination of the expression connectivities" + echo "and spatial connectivities matrices, weighted by \`alpha\`." + echo "" + echo "\$C_{joint} = (1 - \\alpha) \\cdot C_{expression} + \\alpha \\cdot C_{spatial}\$" + echo "" + echo "Inputs:" echo " --input" echo " type: file, required parameter, file must exist" - echo " Path to a single .h5mu file." + echo " Input H5MU file with pre-computed expression and spatial neighborhood" + echo " graphs." echo "" echo " --modality" echo " type: string" echo " default: rna" - echo " Which modality from the input MuData file to process." + echo " Modality from the input MuData file to process." echo "" - echo " --obs_feature" - echo " type: string, required parameter" - echo " example: celltype" - echo " The .obs column to split the mudata on." + echo " --input_obsp_expression_graph" + echo " type: string" + echo " default: connectivities" + echo " Key in \`adata.obsp\` containing the expression connectivity matrix." echo "" - echo " --drop_obs_nan" - echo " type: boolean_true" - echo " Whether to drop all .obs columns that contain only nan values after" - echo " splitting." + echo " --input_obsp_spatial_graph" + echo " type: string" + echo " default: spatial_connectivities" + echo " Key in \`adata.obsp\` containing the spatial connectivity matrix." echo "" - echo " --ensure_unique_filenames" - echo " type: boolean_true" - echo " Append number suffixes to ensure unique filenames after sanitizing obs" - echo " feature values." + echo "Parameters:" + echo " --alpha" + echo " type: double" + echo " default: 0.5" + echo " min: 0.0" + echo " max: 1.0" + echo " Weight of the spatial graph in the connection fusion." + echo " A value of 0 results in the original expression graph;" + echo " a value of 1 results in the spatial graph only." echo "" echo "Outputs:" echo " --output" echo " type: file, required parameter, output, file must exist" - echo " example: /path/to/output" - echo " Output directory containing multiple h5mu files." + echo " example: output.h5mu" + echo " Output H5MU file path." echo "" - echo " --output_files" - echo " type: file, required parameter, output, file must exist" - echo " example: sample_files.csv" - echo " A csv containing the base filename and obs feature by which it was" - echo " split." + echo " --output_obsp_graph" + echo " type: string" + echo " default: spatial_expression_connectivities" + echo " Key under which to store the fused connectivity matrix in \`adata.obsp\`." echo "" echo " --output_compression" echo " type: string" @@ -674,7 +684,7 @@ while [[ $# -gt 0 ]]; do shift 1 ;; --version) - echo "split_h5mu niche-compass" + echo "join_graphs niche-compass" exit ;; --input) @@ -699,25 +709,37 @@ while [[ $# -gt 0 ]]; do VIASH_PAR_MODALITY=$(ViashRemoveFlags "$1") shift 1 ;; - --obs_feature) - [ -n "$VIASH_PAR_OBS_FEATURE" ] && ViashError Bad arguments for option \'--obs_feature\': \'$VIASH_PAR_OBS_FEATURE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OBS_FEATURE="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --obs_feature. Use "--help" to get more information on the parameters. && exit 1 + --input_obsp_expression_graph) + [ -n "$VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH" ] && ViashError Bad arguments for option \'--input_obsp_expression_graph\': \'$VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input_obsp_expression_graph. Use "--help" to get more information on the parameters. && exit 1 shift 2 ;; - --obs_feature=*) - [ -n "$VIASH_PAR_OBS_FEATURE" ] && ViashError Bad arguments for option \'--obs_feature=*\': \'$VIASH_PAR_OBS_FEATURE\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OBS_FEATURE=$(ViashRemoveFlags "$1") + --input_obsp_expression_graph=*) + [ -n "$VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH" ] && ViashError Bad arguments for option \'--input_obsp_expression_graph=*\': \'$VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH=$(ViashRemoveFlags "$1") shift 1 ;; - --drop_obs_nan) - [ -n "$VIASH_PAR_DROP_OBS_NAN" ] && ViashError Bad arguments for option \'--drop_obs_nan\': \'$VIASH_PAR_DROP_OBS_NAN\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_DROP_OBS_NAN=true + --input_obsp_spatial_graph) + [ -n "$VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH" ] && ViashError Bad arguments for option \'--input_obsp_spatial_graph\': \'$VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --input_obsp_spatial_graph. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --input_obsp_spatial_graph=*) + [ -n "$VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH" ] && ViashError Bad arguments for option \'--input_obsp_spatial_graph=*\': \'$VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH=$(ViashRemoveFlags "$1") shift 1 ;; - --ensure_unique_filenames) - [ -n "$VIASH_PAR_ENSURE_UNIQUE_FILENAMES" ] && ViashError Bad arguments for option \'--ensure_unique_filenames\': \'$VIASH_PAR_ENSURE_UNIQUE_FILENAMES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_ENSURE_UNIQUE_FILENAMES=true + --alpha) + [ -n "$VIASH_PAR_ALPHA" ] && ViashError Bad arguments for option \'--alpha\': \'$VIASH_PAR_ALPHA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ALPHA="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --alpha. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --alpha=*) + [ -n "$VIASH_PAR_ALPHA" ] && ViashError Bad arguments for option \'--alpha=*\': \'$VIASH_PAR_ALPHA\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_ALPHA=$(ViashRemoveFlags "$1") shift 1 ;; --output) @@ -731,15 +753,15 @@ while [[ $# -gt 0 ]]; do VIASH_PAR_OUTPUT=$(ViashRemoveFlags "$1") shift 1 ;; - --output_files) - [ -n "$VIASH_PAR_OUTPUT_FILES" ] && ViashError Bad arguments for option \'--output_files\': \'$VIASH_PAR_OUTPUT_FILES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT_FILES="$2" - [ $# -lt 2 ] && ViashError Not enough arguments passed to --output_files. Use "--help" to get more information on the parameters. && exit 1 + --output_obsp_graph) + [ -n "$VIASH_PAR_OUTPUT_OBSP_GRAPH" ] && ViashError Bad arguments for option \'--output_obsp_graph\': \'$VIASH_PAR_OUTPUT_OBSP_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT_OBSP_GRAPH="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --output_obsp_graph. Use "--help" to get more information on the parameters. && exit 1 shift 2 ;; - --output_files=*) - [ -n "$VIASH_PAR_OUTPUT_FILES" ] && ViashError Bad arguments for option \'--output_files=*\': \'$VIASH_PAR_OUTPUT_FILES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 - VIASH_PAR_OUTPUT_FILES=$(ViashRemoveFlags "$1") + --output_obsp_graph=*) + [ -n "$VIASH_PAR_OUTPUT_OBSP_GRAPH" ] && ViashError Bad arguments for option \'--output_obsp_graph=*\': \'$VIASH_PAR_OUTPUT_OBSP_GRAPH\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_OUTPUT_OBSP_GRAPH=$(ViashRemoveFlags "$1") shift 1 ;; --output_compression) @@ -841,7 +863,7 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then # determine docker image id if [[ "$VIASH_ENGINE_ID" == 'docker' ]]; then - VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/dataflow/split_h5mu:niche-compass' + VIASH_DOCKER_IMAGE_ID='images.viash-hub.com/vsh/openpipeline_spatial/neighbors/join_graphs:niche-compass' fi # print dockerfile @@ -929,18 +951,10 @@ if [ -z ${VIASH_PAR_INPUT+x} ]; then ViashError '--input' is a required argument. Use "--help" to get more information on the parameters. exit 1 fi -if [ -z ${VIASH_PAR_OBS_FEATURE+x} ]; then - ViashError '--obs_feature' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi if [ -z ${VIASH_PAR_OUTPUT+x} ]; then ViashError '--output' is a required argument. Use "--help" to get more information on the parameters. exit 1 fi -if [ -z ${VIASH_PAR_OUTPUT_FILES+x} ]; then - ViashError '--output_files' is a required argument. Use "--help" to get more information on the parameters. - exit 1 -fi if [ -z ${VIASH_META_NAME+x} ]; then ViashError 'name' is a required argument. Use "--help" to get more information on the parameters. exit 1 @@ -970,11 +984,17 @@ fi if [ -z ${VIASH_PAR_MODALITY+x} ]; then VIASH_PAR_MODALITY="rna" fi -if [ -z ${VIASH_PAR_DROP_OBS_NAN+x} ]; then - VIASH_PAR_DROP_OBS_NAN="false" +if [ -z ${VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH+x} ]; then + VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH="connectivities" fi -if [ -z ${VIASH_PAR_ENSURE_UNIQUE_FILENAMES+x} ]; then - VIASH_PAR_ENSURE_UNIQUE_FILENAMES="false" +if [ -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH+x} ]; then + VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH="spatial_connectivities" +fi +if [ -z ${VIASH_PAR_ALPHA+x} ]; then + VIASH_PAR_ALPHA="0.5" +fi +if [ -z ${VIASH_PAR_OUTPUT_OBSP_GRAPH+x} ]; then + VIASH_PAR_OUTPUT_OBSP_GRAPH="spatial_expression_connectivities" fi # check whether required files exist @@ -984,16 +1004,36 @@ if [ ! -z "$VIASH_PAR_INPUT" ] && [ ! -e "$VIASH_PAR_INPUT" ]; then fi # check whether parameters values are of the right type -if [[ -n "$VIASH_PAR_DROP_OBS_NAN" ]]; then - if ! [[ "$VIASH_PAR_DROP_OBS_NAN" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then - ViashError '--drop_obs_nan' has to be a boolean_true. Use "--help" to get more information on the parameters. +if [[ -n "$VIASH_PAR_ALPHA" ]]; then + if ! [[ "$VIASH_PAR_ALPHA" =~ ^[-+]?(\.[0-9]+|[0-9]+(\.[0-9]*)?)([eE][-+]?[0-9]+)?$ ]]; then + ViashError '--alpha' has to be a double. Use "--help" to get more information on the parameters. exit 1 fi -fi -if [[ -n "$VIASH_PAR_ENSURE_UNIQUE_FILENAMES" ]]; then - if ! [[ "$VIASH_PAR_ENSURE_UNIQUE_FILENAMES" =~ ^(true|True|TRUE|false|False|FALSE|yes|Yes|YES|no|No|NO)$ ]]; then - ViashError '--ensure_unique_filenames' has to be a boolean_true. Use "--help" to get more information on the parameters. - exit 1 + if command -v bc &> /dev/null; then + if ! [[ `echo $VIASH_PAR_ALPHA '>=' 0.0 | bc` -eq 1 ]]; then + ViashError '--alpha' has be more than or equal to 0.0. Use "--help" to get more information on the parameters. + exit 1 + fi + elif command -v awk &> /dev/null; then + if ! [[ `awk -v n1=$VIASH_PAR_ALPHA -v n2=0.0 'BEGIN { print (n1 >= n2) ? "1" : "0" }'` -eq 1 ]]; then + ViashError '--alpha' has be more than or equal to 0.0. Use "--help" to get more information on the parameters. + exit 1 + fi + else + ViashWarning '--alpha' specifies a minimum value but the value was not verified as neither \'bc\' or \`awk\` are present on the system. + fi + if command -v bc &> /dev/null; then + if ! [[ `echo $VIASH_PAR_ALPHA '<=' 1.0 | bc` -eq 1 ]]; then + ViashError '--alpha' has to be less than or equal to 1.0. Use "--help" to get more information on the parameters. + exit 1 + fi + elif command -v awk &> /dev/null; then + if ! [[ `awk -v n1=$VIASH_PAR_ALPHA -v n2=1.0 'BEGIN { print (n1 <= n2) ? "1" : "0" }'` -eq 1 ]]; then + ViashError '--alpha' has be less than or equal to 1.0. Use "--help" to get more information on the parameters. + exit 1 + fi + else + ViashWarning '--alpha' specifies a maximum value but the value was not verified as neither \'bc\' or \'awk\' are present on the system. fi fi if [[ -n "$VIASH_META_CPUS" ]]; then @@ -1086,9 +1126,6 @@ fi if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT")" ]; then mkdir -p "$(dirname "$VIASH_PAR_OUTPUT")" fi -if [ ! -z "$VIASH_PAR_OUTPUT_FILES" ] && [ ! -d "$(dirname "$VIASH_PAR_OUTPUT_FILES")" ]; then - mkdir -p "$(dirname "$VIASH_PAR_OUTPUT_FILES")" -fi if [ "$VIASH_ENGINE_ID" == "native" ] ; then if [ "$VIASH_MODE" == "run" ]; then @@ -1111,11 +1148,6 @@ if [ ! -z "$VIASH_PAR_OUTPUT" ]; then VIASH_PAR_OUTPUT=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT") VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT" ) fi -if [ ! -z "$VIASH_PAR_OUTPUT_FILES" ]; then - VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_PAR_OUTPUT_FILES")" ) - VIASH_PAR_OUTPUT_FILES=$(ViashDockerAutodetectMount "$VIASH_PAR_OUTPUT_FILES") - VIASH_CHOWN_VARS+=( "$VIASH_PAR_OUTPUT_FILES" ) -fi if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then VIASH_DIRECTORY_MOUNTS+=( "$(ViashDockerAutodetectMountArg "$VIASH_META_RESOURCES_DIR")" ) VIASH_META_RESOURCES_DIR=$(ViashDockerAutodetectMount "$VIASH_META_RESOURCES_DIR") @@ -1172,7 +1204,7 @@ fi ViashDebug "Running command: $(echo $VIASH_CMD)" cat << VIASHEOF | eval $VIASH_CMD set -e -tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-split_h5mu-XXXXXX").py +tempscript=\$(mktemp "$VIASH_META_TEMP_DIR/viash-run-join_graphs-XXXXXX").py function clean_up { rm "\$tempscript" } @@ -1185,22 +1217,17 @@ trap interrupt INT SIGINT cat > "\$tempscript" << 'VIASHMAIN' import sys import mudata as mu -import pandas as pd -import re -import gc -from pathlib import Path -from collections import defaultdict -### VIASH START +## VIASH START # The following code has been auto-generated by Viash. par = { 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'obs_feature': $( if [ ! -z ${VIASH_PAR_OBS_FEATURE+x} ]; then echo "r'${VIASH_PAR_OBS_FEATURE//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'drop_obs_nan': $( if [ ! -z ${VIASH_PAR_DROP_OBS_NAN+x} ]; then echo "r'${VIASH_PAR_DROP_OBS_NAN//\'/\'\"\'\"r\'}'.lower() == 'true'"; else echo None; fi ), - 'ensure_unique_filenames': $( if [ ! -z ${VIASH_PAR_ENSURE_UNIQUE_FILENAMES+x} ]; then echo "r'${VIASH_PAR_ENSURE_UNIQUE_FILENAMES//\'/\'\"\'\"r\'}'.lower() == 'true'"; else echo None; fi ), + 'input_obsp_expression_graph': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'input_obsp_spatial_graph': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'alpha': $( if [ ! -z ${VIASH_PAR_ALPHA+x} ]; then echo "float(r'${VIASH_PAR_ALPHA//\'/\'\"\'\"r\'}')"; else echo None; fi ), 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), - 'output_files': $( if [ ! -z ${VIASH_PAR_OUTPUT_FILES+x} ]; then echo "r'${VIASH_PAR_OUTPUT_FILES//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'output_obsp_graph': $( if [ ! -z ${VIASH_PAR_OUTPUT_OBSP_GRAPH+x} ]; then echo "r'${VIASH_PAR_OUTPUT_OBSP_GRAPH//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'output_compression': $( if [ ! -z ${VIASH_PAR_OUTPUT_COMPRESSION+x} ]; then echo "r'${VIASH_PAR_OUTPUT_COMPRESSION//\'/\'\"\'\"r\'}'"; else echo None; fi ) } meta = { @@ -1227,91 +1254,51 @@ dep = { } -### VIASH END +## VIASH END sys.path.append(meta["resources_dir"]) from setup_logger import setup_logger logger = setup_logger() +## Read data +logger.info("Reading input data...") +adata = mu.read_h5ad(par["input"], mod=par["modality"]) -def main(): - logger.info(f"Reading {par['input']}") - input_file = Path(par["input"].strip()) +## Validate inputs +spatial_key = par["input_obsp_spatial_graph"] +if spatial_key not in adata.obsp: + raise ValueError(f"Spatial graph key '{spatial_key}' not found in .obsp.") - mdata = mu.read_h5mu(input_file) - adata = mdata.mod[par["modality"]] +expr_key = par["input_obsp_expression_graph"] +if expr_key not in adata.obsp: + raise ValueError(f"Expression graph key '{expr_key}' not found in .obsp.") - logger.info(f"Reading unique features from {par['obs_feature']}") - obs_features = adata.obs[par["obs_feature"]].unique().tolist() +nn_graph_genes = adata.obsp[expr_key] +nn_graph_space = adata.obsp[spatial_key] - # sanitize --obs_feature values - obs_features_s = [re.sub(r"[-\\s]", "_", str(s).strip()) for s in obs_features] - obs_features_s = [re.sub(r"[^A-Za-z0-9_]", "", s) for s in obs_features_s] +## Combine graphs +alpha = par["alpha"] +logger.info( + f"Combining graphs (alpha={alpha}: spatial weight, " + f"{1 - alpha:.2f}: expression weight)..." +) +joint_graph = (1 - alpha) * nn_graph_genes + alpha * nn_graph_space +out_key = par["output_obsp_graph"] +logger.info(f"Storing result in .obsp['{out_key}']...") +adata.obsp[out_key] = joint_graph +adata.uns[out_key] = { + "params": {"alpha": alpha}, + "inputs": { + "expression_graph": expr_key, + "spatial_graph": spatial_key, + }, +} - # ensure that names are unique, if not raise or append number as suffix - if not len(obs_features_s) == len(set(obs_features_s)): - if not par["ensure_unique_filenames"]: - raise ValueError( - f"File names are not unique after sanitizing the --obs_feature {par['obs_feature']} values" - ) - - logger.info("Ensuring unique names for par['obs_feature']") - counts = defaultdict(lambda: -1) - for i, feature in enumerate(obs_features_s): - counts[feature] += 1 - if (curr_counts := counts[feature]) > 0: - obs_features_s[i] += f"_{curr_counts}" - - # generate output dir - output_dir = Path(par["output"]) - if not output_dir.is_dir(): - output_dir.mkdir(parents=True) - - # split modality of mdata file base on obs_feature - logger.info(f"Splitting file based on {par['obs_feature']} values {obs_features}") - obs_files = [] - - for obs_name, file_name in zip(obs_features, obs_features_s): - logger.info( - f"Filtering modality '{par['modality']}' observations by .obs['{par['obs_feature']}'] == {obs_name}" - ) - mdata_obs = mdata.copy() - adata_full = mdata_obs.mod[par["modality"]] - - # split the samples - mask = adata_full.obs[par["obs_feature"]] == obs_name - adata_obs = adata_full[mask].copy() - - # Dropping columns that only have nan values after splitting - if par["drop_obs_nan"]: - logger.info("Dropping all .obs columns with NaN values") - adata_obs.obs = adata_obs.obs.dropna(axis=1, how="all") - - mdata_obs.mod[par["modality"]] = adata_obs - - mdata_obs_name = f"{input_file.stem}_{file_name}.h5mu" - out_path = output_dir / mdata_obs_name - - # replace mdata file with modality adata contianing split samples - logger.info( - f"Writing h5mu filtered for {par['obs_feature']} {obs_name} to file {out_path}" - ) - - mdata_obs.write_h5mu(out_path, compression=par["output_compression"]) - - # avoid keeping files in memory - obs_files.append(mdata_obs_name) - del mdata_obs, adata_obs - gc.collect() - - logger.info(f"Writing output_files CSV file to {par['output_files']}") - df = pd.DataFrame({"name": obs_features_s, "filename": obs_files}) - df.to_csv(par["output_files"], index=False) - - -if __name__ == "__main__": - main() +## Write output +logger.info("Saving output data...") +mdata = mu.MuData({par["modality"]: adata}) +mdata.write_h5mu(par["output"], compression=par["output_compression"]) VIASHMAIN python -B "\$tempscript" & wait "\$!" @@ -1328,9 +1315,6 @@ if [[ "$VIASH_ENGINE_TYPE" == "docker" ]]; then if [ ! -z "$VIASH_PAR_OUTPUT" ]; then VIASH_PAR_OUTPUT=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT") fi - if [ ! -z "$VIASH_PAR_OUTPUT_FILES" ]; then - VIASH_PAR_OUTPUT_FILES=$(ViashDockerStripAutomount "$VIASH_PAR_OUTPUT_FILES") - fi if [ ! -z "$VIASH_META_RESOURCES_DIR" ]; then VIASH_META_RESOURCES_DIR=$(ViashDockerStripAutomount "$VIASH_META_RESOURCES_DIR") fi @@ -1351,10 +1335,6 @@ if [ ! -z "$VIASH_PAR_OUTPUT" ] && [ ! -e "$VIASH_PAR_OUTPUT" ]; then ViashError "Output file '$VIASH_PAR_OUTPUT' does not exist." exit 1 fi -if [ ! -z "$VIASH_PAR_OUTPUT_FILES" ] && [ ! -e "$VIASH_PAR_OUTPUT_FILES" ]; then - ViashError "Output file '$VIASH_PAR_OUTPUT_FILES' does not exist." - exit 1 -fi exit 0 diff --git a/target/executable/neighbors/join_graphs/nextflow_labels.config b/target/executable/neighbors/join_graphs/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/executable/neighbors/join_graphs/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/executable/neighbors/join_graphs/setup_logger.py b/target/executable/neighbors/join_graphs/setup_logger.py new file mode 100644 index 0000000..3ca1cdb --- /dev/null +++ b/target/executable/neighbors/join_graphs/setup_logger.py @@ -0,0 +1,12 @@ +def setup_logger(): + import logging + from sys import stdout + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + console_handler = logging.StreamHandler(stdout) + logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s") + console_handler.setFormatter(logFormatter) + logger.addHandler(console_handler) + + return logger diff --git a/target/executable/neighbors/spatial_neighborhood_graph/.config.vsh.yaml b/target/executable/neighbors/spatial_neighborhood_graph/.config.vsh.yaml index 9b64492..11d80fe 100644 --- a/target/executable/neighbors/spatial_neighborhood_graph/.config.vsh.yaml +++ b/target/executable/neighbors/spatial_neighborhood_graph/.config.vsh.yaml @@ -147,7 +147,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -270,7 +270,7 @@ build_info: output: "target/executable/neighbors/spatial_neighborhood_graph" executable: "target/executable/neighbors/spatial_neighborhood_graph/spatial_neighborhood_graph" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -284,7 +284,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/neighbors/spatial_neighborhood_graph/spatial_neighborhood_graph b/target/executable/neighbors/spatial_neighborhood_graph/spatial_neighborhood_graph index 2056487..ee3bb32 100755 --- a/target/executable/neighbors/spatial_neighborhood_graph/spatial_neighborhood_graph +++ b/target/executable/neighbors/spatial_neighborhood_graph/spatial_neighborhood_graph @@ -458,9 +458,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component neighbors spatial_neighborhood_graph" -LABEL org.opencontainers.image.created="2026-02-17T11:00:57Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER diff --git a/target/executable/nichecompass/nichecompass/.config.vsh.yaml b/target/executable/nichecompass/nichecompass/.config.vsh.yaml index 6a4d25c..7c580db 100644 --- a/target/executable/nichecompass/nichecompass/.config.vsh.yaml +++ b/target/executable/nichecompass/nichecompass/.config.vsh.yaml @@ -64,6 +64,17 @@ argument_groups: direction: "input" multiple: false multiple_sep: ";" + - type: "string" + name: "--var_input" + description: "Key in adata.var that indicates which features to include as input\ + \ for the NicheCompass model.\nIf not provided, all features will be included.\n" + info: null + default: + - "filter_with_hvg" + required: false + direction: "input" + multiple: false + multiple_sep: ";" - type: "string" name: "--input_obsp_spatial_connectivities" description: "Key in adata.obsp where connectivities of the spatial neighborhood\ @@ -851,6 +862,8 @@ resources: is_executable: true - type: "file" path: "setup_logger.py" +- type: "file" + path: "subset_vars.py" - type: "file" path: "nextflow_labels.config" dest: "nextflow_labels.config" @@ -875,7 +888,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -998,7 +1011,7 @@ build_info: output: "target/executable/nichecompass/nichecompass" executable: "target/executable/nichecompass/nichecompass/nichecompass" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -1012,7 +1025,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/executable/nichecompass/nichecompass/nichecompass b/target/executable/nichecompass/nichecompass/nichecompass index e72721e..b7c4d46 100755 --- a/target/executable/nichecompass/nichecompass/nichecompass +++ b/target/executable/nichecompass/nichecompass/nichecompass @@ -460,9 +460,9 @@ RUN pip install --upgrade pip && \ LABEL org.opencontainers.image.authors="Dorien Roosen" LABEL org.opencontainers.image.description="Companion container for running component nichecompass nichecompass" -LABEL org.opencontainers.image.created="2026-02-17T11:00:58Z" +LABEL org.opencontainers.image.created="2026-03-25T10:11:21Z" LABEL org.opencontainers.image.source="https://github.com/openpipelines-bio/openpipeline_spatial" -LABEL org.opencontainers.image.revision="ad19aba349736848d99a9646870d8e7868eb6515" +LABEL org.opencontainers.image.revision="87e62605aafd706d539bf2978ef47ede6fe41926" LABEL org.opencontainers.image.version="niche-compass" VIASHDOCKER @@ -612,6 +612,13 @@ function ViashHelp { echo " type: string" echo " Input layer to use. If None, X is used" echo "" + echo " --var_input" + echo " type: string" + echo " default: filter_with_hvg" + echo " Key in adata.var that indicates which features to include as input for" + echo " the NicheCompass model." + echo " If not provided, all features will be included." + echo "" echo " --input_obsp_spatial_connectivities" echo " type: string" echo " default: spatial_connectivities" @@ -1169,6 +1176,17 @@ while [[ $# -gt 0 ]]; do VIASH_PAR_LAYER=$(ViashRemoveFlags "$1") shift 1 ;; + --var_input) + [ -n "$VIASH_PAR_VAR_INPUT" ] && ViashError Bad arguments for option \'--var_input\': \'$VIASH_PAR_VAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_VAR_INPUT="$2" + [ $# -lt 2 ] && ViashError Not enough arguments passed to --var_input. Use "--help" to get more information on the parameters. && exit 1 + shift 2 + ;; + --var_input=*) + [ -n "$VIASH_PAR_VAR_INPUT" ] && ViashError Bad arguments for option \'--var_input=*\': \'$VIASH_PAR_VAR_INPUT\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 + VIASH_PAR_VAR_INPUT=$(ViashRemoveFlags "$1") + shift 1 + ;; --input_obsp_spatial_connectivities) [ -n "$VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES" ] && ViashError Bad arguments for option \'--input_obsp_spatial_connectivities\': \'$VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES\' \& \'$2\' - you should provide exactly one argument for this option. && exit 1 VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES="$2" @@ -2137,6 +2155,9 @@ fi if [ -z ${VIASH_PAR_MODALITY+x} ]; then VIASH_PAR_MODALITY="rna" fi +if [ -z ${VIASH_PAR_VAR_INPUT+x} ]; then + VIASH_PAR_VAR_INPUT="filter_with_hvg" +fi if [ -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES+x} ]; then VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES="spatial_connectivities" fi @@ -3120,6 +3141,7 @@ par = { 'input_gp_mask': $( if [ ! -z ${VIASH_PAR_INPUT_GP_MASK+x} ]; then echo "r'${VIASH_PAR_INPUT_GP_MASK//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'layer': $( if [ ! -z ${VIASH_PAR_LAYER+x} ]; then echo "r'${VIASH_PAR_LAYER//\'/\'\"\'\"r\'}'"; else echo None; fi ), + 'var_input': $( if [ ! -z ${VIASH_PAR_VAR_INPUT+x} ]; then echo "r'${VIASH_PAR_VAR_INPUT//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'input_obsp_spatial_connectivities': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES//\'/\'\"\'\"r\'}'"; else echo None; fi ), 'input_obs_covariates': $( if [ ! -z ${VIASH_PAR_INPUT_OBS_COVARIATES+x} ]; then echo "r'${VIASH_PAR_INPUT_OBS_COVARIATES//\'/\'\"\'\"r\'}'.split(';')"; else echo None; fi ), 'min_genes_per_gp': $( if [ ! -z ${VIASH_PAR_MIN_GENES_PER_GP+x} ]; then echo "int(r'${VIASH_PAR_MIN_GENES_PER_GP//\'/\'\"\'\"r\'}')"; else echo None; fi ), @@ -3214,6 +3236,7 @@ dep = { sys.path.append(meta["resources_dir"]) from setup_logger import setup_logger +from subset_vars import subset_vars logger = setup_logger() @@ -3226,6 +3249,11 @@ logger.info(f"GPU count: {torch.cuda.device_count()}") ## Read in data adata = mu.read_h5ad(par["input"], mod=par["modality"]) +# Subset to HVG +if par["var_input"]: + # Subset to HVG + adata = subset_vars(adata, subset_col=par["var_input"]).copy() + # Counts need to be float32 to be processed by nichecompass model # See https://discuss.pytorch.org/t/runtimeerror-mat1-and-mat2-must-have-the-same-dtype/166759 counts_dtype = ( diff --git a/target/executable/nichecompass/nichecompass/subset_vars.py b/target/executable/nichecompass/nichecompass/subset_vars.py new file mode 100644 index 0000000..4025000 --- /dev/null +++ b/target/executable/nichecompass/nichecompass/subset_vars.py @@ -0,0 +1,49 @@ +def subset_vars(adata, subset_col): + """ + Subset AnnData object on highly variable genes or a boolean mask. + + Parameters + ---------- + adata : AnnData + Annotated data object + subset_col : str, pd.Series, pd.Index, or np.ndarray + Name of the boolean column in `adata.var` that contains the information if features should be used or not, + or a boolean mask (same length as adata.var) + + Returns + ------- + AnnData + Copy of `adata` with subsetted features + """ + import pandas as pd + import numpy as np + + # Convert all input types to a pandas Series + if isinstance(subset_col, str): + if subset_col not in adata.var.columns: + raise ValueError( + f"Requested to use .var column '{subset_col}' as a selection of genes, but the column is not available." + ) + mask = adata.var[subset_col] + elif isinstance(subset_col, pd.Series): + mask = subset_col + elif isinstance(subset_col, (pd.Index, np.ndarray, list)): + mask = pd.Series(subset_col, index=adata.var.index) + else: + raise TypeError( + "subset_col must be a string (column name) or a boolean mask (Series, Index, ndarray, or list)." + ) + + # Validate mask + if not pd.api.types.is_bool_dtype(mask): + raise ValueError( + f"Expected mask to be boolean, but found {mask.dtype}. Can not subset data." + ) + if mask.isna().sum() > 0: + raise ValueError("Mask contains NaN values. Can not subset data.") + if len(mask) != adata.n_vars: + raise ValueError( + f"Mask length {len(mask)} does not match number of variables {adata.n_vars}." + ) + + return adata[:, mask].copy() diff --git a/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml index 410ae85..196e46c 100644 --- a/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_cells2stats_to_h5mu/.config.vsh.yaml @@ -179,7 +179,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -303,7 +303,7 @@ build_info: output: "target/nextflow/convert/from_cells2stats_to_h5mu" executable: "target/nextflow/convert/from_cells2stats_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -317,7 +317,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf b/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf index 834c628..2375d9e 100644 --- a/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf +++ b/target/nextflow/convert/from_cells2stats_to_h5mu/main.nf @@ -3246,7 +3246,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3400,7 +3400,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cells2stats_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3420,7 +3420,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml index efc09fa..e7e270b 100644 --- a/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_cosmx_to_h5mu/.config.vsh.yaml @@ -107,7 +107,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -239,7 +239,7 @@ build_info: output: "target/nextflow/convert/from_cosmx_to_h5mu" executable: "target/nextflow/convert/from_cosmx_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -253,7 +253,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_cosmx_to_h5mu/main.nf b/target/nextflow/convert/from_cosmx_to_h5mu/main.nf index 2d67467..9f7cf4d 100644 --- a/target/nextflow/convert/from_cosmx_to_h5mu/main.nf +++ b/target/nextflow/convert/from_cosmx_to_h5mu/main.nf @@ -3184,7 +3184,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3350,7 +3350,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cosmx_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3370,7 +3370,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml index 4176cf8..3fbd32e 100644 --- a/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml @@ -125,7 +125,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -234,7 +234,7 @@ build_info: output: "target/nextflow/convert/from_cosmx_to_spatialexperiment" executable: "target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -248,7 +248,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf b/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf index 2875a3e..a26c7f8 100644 --- a/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_cosmx_to_spatialexperiment/main.nf @@ -3190,7 +3190,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3326,7 +3326,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_cosmx_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3346,7 +3346,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/.config.vsh.yaml b/target/nextflow/convert/from_h5mu_to_spatialdata/.config.vsh.yaml new file mode 100644 index 0000000..0e2f773 --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/.config.vsh.yaml @@ -0,0 +1,268 @@ +name: "from_h5mu_to_spatialdata" +namespace: "convert" +version: "niche-compass" +authors: +- name: "Dorien Roosen" + roles: + - "maintainer" + info: + role: "Core Team Member" + links: + email: "dorien@data-intuitive.com" + github: "dorien-er" + linkedin: "dorien-roosen" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Data Scientist" +- name: "Luke Zappia" + roles: + - "author" + info: + role: "Contributor" + links: + email: "luke@data-intuitive.com" + github: "lazappi" + orcid: "0000-0001-7744-8565" + linkedin: "lazappi" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Data Science Engineer" +argument_groups: +- name: "Arguments" + arguments: + - type: "file" + name: "--input" + alternatives: + - "-i" + description: "Input H5MU file." + info: null + example: + - "input.h5mu" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--input_spatialdata" + description: "An optional existing SpatialData Zarr store to fill remaining slots\ + \ from." + info: null + example: + - "existing.zarr" + must_exist: true + create_parent: true + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "The modality in the MuData to be used as the main table in the SpatialData\ + \ object." + info: null + default: + - "rna" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "file" + name: "--output" + alternatives: + - "-o" + description: "The path to the output SpatialData Zarr store." + info: null + example: + - "output.zarr" + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "setup_logger.py" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +description: "Reads in an H5MU file and saves it as a SpatialData Zarr store. The\ + \ selected\nmodality in the MuData is stored as the main table in the SpatialData\ + \ object.\nIf a matching existing SpatialData is provided, it will be used to fill\ + \ the\nremaining SpatialData slots.\n \n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.h5mu" +- type: "file" + path: "xenium_tiny.zarr" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "lowmem" + - "singlecpu" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.12-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" + - "pyarrow~=18.0.0" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "apt" + packages: + - "git" + interactive: false + - type: "python" + user: false + packages: + - "viashpy==0.9.0" + github: + - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/convert/from_h5mu_to_spatialdata/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/convert/from_h5mu_to_spatialdata" + executable: "target/nextflow/convert/from_h5mu_to_spatialdata/main.nf" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/main.nf b/target/nextflow/convert/from_h5mu_to_spatialdata/main.nf new file mode 100644 index 0000000..e1d4c5f --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/main.nf @@ -0,0 +1,3986 @@ +// from_h5mu_to_spatialdata niche-compass +// +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +// Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Dorien Roosen (maintainer) +// * Luke Zappia (author) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be. only cast if the value is a GString + if (value instanceof GString) { + value = value as String + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value !instanceof Integer) { + try { + value = value as Integer + } catch (NumberFormatException e) { + expectedClass = "Integer" + } + } + } else if (par.type == "long") { + // cast to long if need be + if (value !instanceof Long) { + try { + value = value as Long + } catch (NumberFormatException e) { + expectedClass = "Long" + } + } + } else if (par.type == "double") { + // cast to double if need be + if (value !instanceof Double) { + try { + value = value as Double + } catch (NumberFormatException e) { + expectedClass = "Double" + } + } + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } + } + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" + } + } + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } + } + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required && arg.direction == "input") { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile) + output: + tuple val(id), path{[yamlFile]} + script: + """ + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - (key, value) are the tuples that will be saved to the state.yaml file + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + return value_ + } + return [["key": plainName_, "value": outputPerFile]] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [["key": plainName_, value: value_]] + } + } + + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutputMulti = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) + + [join_id, channelId, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chJoined, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublishStates = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublishStates, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "from_h5mu_to_spatialdata", + "namespace" : "convert", + "version" : "niche-compass", + "authors" : [ + { + "name" : "Dorien Roosen", + "roles" : [ + "maintainer" + ], + "info" : { + "role" : "Core Team Member", + "links" : { + "email" : "dorien@data-intuitive.com", + "github" : "dorien-er", + "linkedin" : "dorien-roosen" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Data Scientist" + } + ] + } + }, + { + "name" : "Luke Zappia", + "roles" : [ + "author" + ], + "info" : { + "role" : "Contributor", + "links" : { + "email" : "luke@data-intuitive.com", + "github" : "lazappi", + "orcid" : "0000-0001-7744-8565", + "linkedin" : "lazappi" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Data Science Engineer" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Arguments", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "alternatives" : [ + "-i" + ], + "description" : "Input H5MU file.", + "example" : [ + "input.h5mu" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--input_spatialdata", + "description" : "An optional existing SpatialData Zarr store to fill remaining slots from.", + "example" : [ + "existing.zarr" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--modality", + "description" : "The modality in the MuData to be used as the main table in the SpatialData object.", + "default" : [ + "rna" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "file", + "name" : "--output", + "alternatives" : [ + "-o" + ], + "description" : "The path to the output SpatialData Zarr store.", + "example" : [ + "output.zarr" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "python_script", + "path" : "script.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/utils/setup_logger.py" + }, + { + "type" : "file", + "path" : "/src/workflows/utils/labels.config", + "dest" : "nextflow_labels.config" + } + ], + "description" : "Reads in an H5MU file and saves it as a SpatialData Zarr store. The selected\nmodality in the MuData is stored as the main table in the SpatialData object.\nIf a matching existing SpatialData is provided, it will be used to fill the\nremaining SpatialData slots.\n \n", + "test_resources" : [ + { + "type" : "python_script", + "path" : "test.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/resources_test/xenium/xenium_tiny.h5mu" + }, + { + "type" : "file", + "path" : "/resources_test/xenium/xenium_tiny.zarr" + } + ], + "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "label" : [ + "lowmem", + "singlecpu" + ], + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "python:3.12-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "niche-compass", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "procps" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "anndata~=0.12.7", + "awkward", + "mudata~=0.3.2", + "spatialdata~=0.7.2", + "ome-zarr~=0.12.2", + "pyarrow~=18.0.0" + ], + "script" : [ + "exec(\\"try:\\\\n import zarr; from importlib.metadata import version\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: assert int(version(\\\\\\"zarr\\\\\\").partition(\\\\\\".\\\\\\")[0]) > 2\\")" + ], + "upgrade" : true + } + ], + "test_setup" : [ + { + "type" : "apt", + "packages" : [ + "git" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "viashpy==0.9.0" + ], + "github" : [ + "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" + ], + "upgrade" : true + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/convert/from_h5mu_to_spatialdata/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/convert/from_h5mu_to_spatialdata", + "viash_version" : "0.9.4", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", + "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" + }, + "package_config" : { + "name" : "openpipeline_spatial", + "version" : "niche-compass", + "info" : { + "test_resources" : [ + { + "type" : "s3", + "path" : "s3://openpipelines-bio/openpipeline_spatial/resources_test", + "dest" : "resources_test" + } + ] + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "viash_version" : "0.9.4", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", + "config_mods" : [ + ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'niche-compass'" + ], + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.py" +cat > "$tempscript" << VIASHMAIN +import logging +import sys + +import mudata as mu +import spatialdata as sd + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'input_spatialdata': $( if [ ! -z ${VIASH_PAR_INPUT_SPATIALDATA+x} ]; then echo "r'${VIASH_PAR_INPUT_SPATIALDATA//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + +sys.path.append(meta["resources_dir"]) +from setup_logger import setup_logger + +logger = setup_logger() + +logger.info("Starting conversion from H5MU to SpatialData...") + +logger.info(f"Reading input H5MU file from {par['input']}...") +mdata = mu.read_h5mu(par["input"]) + +logger.info("Extracting modality from MuData object...") +mod = mdata.mod[par["modality"]] + +if par.get("input_spatialdata", None) is not None: + logger.info(f"Reading existing SpatialData from {par['input_spatialdata']}...") + + # Disable logger messages from spatialdata when reading + logger.setLevel(logging.WARNING) + sdata_existing = sd.read_zarr(par["input_spatialdata"]) + logger.setLevel(logging.INFO) + + logger.info("Checking modality matches existing SpatialData table...") + if not mod.n_obs == sdata_existing["table"].n_obs: + raise ValueError( + "The number of observations in the selected modality does not match the existing SpatialData table." + ) + if not mod.obs_names.equals(sdata_existing["table"].obs_names): + raise ValueError( + "The observation names in the selected modality do not match the existing SpatialData table." + ) + +logger.info("Creating SpatialData object...") +if par.get("input_spatialdata", None) is not None: + logger.info("Using existing SpatialData...") + sdata = sdata_existing + sdata["table"] = mod +else: + logger.info("Creating new SpatialData...") + sdata = sd.SpatialData(tables={"table": mod}) + +logger.info(f"Writing output SpatialData Zarr store to {par['output']}...") +sdata.write(par["output"], overwrite=True) + +logger.info("Done!") +VIASHMAIN +python -B "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val + .replaceAll('\\$id', id) + .replaceAll('\\$\\{id\\}', id) + .replaceAll('\\$key', key) + .replaceAll('\\$\\{key\\}', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = _getScriptLoader(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline_spatial/convert/from_h5mu_to_spatialdata", + "tag" : "niche-compass" + }, + "label" : [ + "lowmem", + "singlecpu" + ], + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow.config b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow.config new file mode 100644 index 0000000..c2f0838 --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'convert/from_h5mu_to_spatialdata' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'niche-compass' + description = 'Reads in an H5MU file and saves it as a SpatialData Zarr store. The selected\nmodality in the MuData is stored as the main table in the SpatialData object.\nIf a matching existing SpatialData is provided, it will be used to fill the\nremaining SpatialData slots.\n \n' + author = 'Dorien Roosen, Luke Zappia' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_labels.config b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_schema.json b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_schema.json new file mode 100644 index 0000000..a0a6827 --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/nextflow_schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "from_h5mu_to_spatialdata", + "description": "Reads in an H5MU file and saves it as a SpatialData Zarr store. The selected\nmodality in the MuData is stored as the main table in the SpatialData object.\nIf a matching existing SpatialData is provided, it will be used to fill the\nremaining SpatialData slots.\n \n", + "type": "object", + "$defs": { + "arguments": { + "title": "Arguments", + "type": "object", + "description": "No description", + "properties": { + "input": { + "type": "string", + "format": "path", + "exists": true, + "description": "Input H5MU file.", + "help_text": "Type: `file`, multiple: `False`, required, direction: `input`, example: `\"input.h5mu\"`. " + }, + "input_spatialdata": { + "type": "string", + "format": "path", + "description": "An optional existing SpatialData Zarr store to fill remaining slots from.", + "help_text": "Type: `file`, multiple: `False`, direction: `input`, example: `\"existing.zarr\"`. " + }, + "modality": { + "type": "string", + "description": "The modality in the MuData to be used as the main table in the SpatialData object.", + "help_text": "Type: `string`, multiple: `False`, default: `\"rna\"`. ", + "default": "rna" + }, + "output": { + "type": "string", + "format": "path", + "description": "The path to the output SpatialData Zarr store.", + "help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output.zarr\"`, direction: `output`, example: `\"output.zarr\"`. ", + "default": "$id.$key.output.zarr" + } + } + }, + "nextflow input-output arguments": { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + "publish_dir": { + "type": "string", + "description": "Path to an output directory.", + "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/arguments" + }, + { + "$ref": "#/$defs/nextflow input-output arguments" + } + ] +} diff --git a/target/nextflow/convert/from_h5mu_to_spatialdata/setup_logger.py b/target/nextflow/convert/from_h5mu_to_spatialdata/setup_logger.py new file mode 100644 index 0000000..3ca1cdb --- /dev/null +++ b/target/nextflow/convert/from_h5mu_to_spatialdata/setup_logger.py @@ -0,0 +1,12 @@ +def setup_logger(): + import logging + from sys import stdout + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + console_handler = logging.StreamHandler(stdout) + logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s") + console_handler.setFormatter(logFormatter) + logger.addHandler(console_handler) + + return logger diff --git a/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml index 97a4bec..4ed637f 100644 --- a/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml @@ -92,7 +92,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -198,6 +198,7 @@ engines: - "python3-pip" - "python3-dev" - "python-is-python3" + - "cmake" interactive: false - type: "r" cran: @@ -227,7 +228,7 @@ build_info: output: "target/nextflow/convert/from_h5mu_to_spatialexperiment" executable: "target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -241,7 +242,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf b/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf index 5587164..8e078ab 100644 --- a/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_h5mu_to_spatialexperiment/main.nf @@ -3163,7 +3163,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3295,7 +3295,8 @@ meta = [ "python3", "python3-pip", "python3-dev", - "python-is-python3" + "python-is-python3", + "cmake" ], "interactive" : false }, @@ -3334,7 +3335,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_h5mu_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3354,7 +3355,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_spaceranger_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_spaceranger_to_h5mu/.config.vsh.yaml index 2867e58..af0d2e4 100644 --- a/target/nextflow/convert/from_spaceranger_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_spaceranger_to_h5mu/.config.vsh.yaml @@ -144,7 +144,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -265,7 +265,7 @@ build_info: output: "target/nextflow/convert/from_spaceranger_to_h5mu" executable: "target/nextflow/convert/from_spaceranger_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -279,7 +279,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_spaceranger_to_h5mu/main.nf b/target/nextflow/convert/from_spaceranger_to_h5mu/main.nf index 602297b..f6ef4a8 100644 --- a/target/nextflow/convert/from_spaceranger_to_h5mu/main.nf +++ b/target/nextflow/convert/from_spaceranger_to_h5mu/main.nf @@ -3217,7 +3217,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3368,7 +3368,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_spaceranger_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3388,7 +3388,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml index d04adc1..f3328bf 100644 --- a/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_spatialdata_to_h5mu/.config.vsh.yaml @@ -101,7 +101,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -194,6 +194,7 @@ engines: - "awkward" - "mudata~=0.3.2" - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" - "pyarrow~=18.0.0" script: - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ @@ -223,7 +224,7 @@ build_info: output: "target/nextflow/convert/from_spatialdata_to_h5mu" executable: "target/nextflow/convert/from_spatialdata_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -237,7 +238,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf b/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf index e54d4e6..38f03c8 100644 --- a/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf +++ b/target/nextflow/convert/from_spatialdata_to_h5mu/main.nf @@ -3180,7 +3180,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3292,6 +3292,7 @@ meta = [ "awkward", "mudata~=0.3.2", "spatialdata~=0.7.2", + "ome-zarr~=0.12.2", "pyarrow~=18.0.0" ], "script" : [ @@ -3332,7 +3333,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_spatialdata_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3352,7 +3353,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml index 7fda1e6..5e3ccc8 100644 --- a/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_h5mu/.config.vsh.yaml @@ -121,7 +121,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -246,7 +246,7 @@ build_info: output: "target/nextflow/convert/from_xenium_to_h5mu" executable: "target/nextflow/convert/from_xenium_to_h5mu/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -260,7 +260,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_xenium_to_h5mu/main.nf b/target/nextflow/convert/from_xenium_to_h5mu/main.nf index 943ac34..922c2fe 100644 --- a/target/nextflow/convert/from_xenium_to_h5mu/main.nf +++ b/target/nextflow/convert/from_xenium_to_h5mu/main.nf @@ -3194,7 +3194,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3349,7 +3349,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_h5mu", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3369,7 +3369,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml index d6cff9c..318d861 100644 --- a/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_spatialdata/.config.vsh.yaml @@ -201,7 +201,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -295,9 +295,14 @@ engines: packages: - "spatialdata-io~=0.5.1" - "spatialdata~=0.7.2" + - "ome-zarr~=0.12.2" - "pyarrow~=18.0.0" git: - "https://codeberg.org/miurahr/zipfile-inflate64.git@v0.2" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" upgrade: true test_setup: - type: "apt" @@ -326,7 +331,7 @@ build_info: output: "target/nextflow/convert/from_xenium_to_spatialdata" executable: "target/nextflow/convert/from_xenium_to_spatialdata/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -340,7 +345,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_xenium_to_spatialdata/main.nf b/target/nextflow/convert/from_xenium_to_spatialdata/main.nf index 7fbe4de..4d4b9de 100644 --- a/target/nextflow/convert/from_xenium_to_spatialdata/main.nf +++ b/target/nextflow/convert/from_xenium_to_spatialdata/main.nf @@ -3283,7 +3283,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3396,11 +3396,15 @@ meta = [ "packages" : [ "spatialdata-io~=0.5.1", "spatialdata~=0.7.2", + "ome-zarr~=0.12.2", "pyarrow~=18.0.0" ], "git" : [ "https://codeberg.org/miurahr/zipfile-inflate64.git@v0.2" ], + "script" : [ + "exec(\\"try:\\\\n import zarr; from importlib.metadata import version\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: assert int(version(\\\\\\"zarr\\\\\\").partition(\\\\\\".\\\\\\")[0]) > 2\\")" + ], "upgrade" : true } ], @@ -3443,7 +3447,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_spatialdata", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3463,7 +3467,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml b/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml index fd995b6..7a1eee2 100644 --- a/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml +++ b/target/nextflow/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml @@ -115,7 +115,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -210,12 +210,6 @@ engines: - "install.packages(\"arrow\", type = \"source\")" bioc_force_install: false warnings_as_errors: true - - type: "r" - script: - - "Sys.setenv(LIBARROW_MINIMAL = \"false\"); install.packages(\"arrow\", type\ - \ = \"source\")" - bioc_force_install: false - warnings_as_errors: true - type: "r" bioc: - "SpatialExperimentIO" @@ -238,7 +232,7 @@ build_info: output: "target/nextflow/convert/from_xenium_to_spatialexperiment" executable: "target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -252,7 +246,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf b/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf index 4e303fe..e3ee550 100644 --- a/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf +++ b/target/nextflow/convert/from_xenium_to_spatialexperiment/main.nf @@ -3179,7 +3179,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3298,14 +3298,6 @@ meta = [ "bioc_force_install" : false, "warnings_as_errors" : true }, - { - "type" : "r", - "script" : [ - "Sys.setenv(LIBARROW_MINIMAL = \\"false\\"); install.packages(\\"arrow\\", type = \\"source\\")" - ], - "bioc_force_install" : false, - "warnings_as_errors" : true - }, { "type" : "r", "bioc" : [ @@ -3337,7 +3329,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/convert/from_xenium_to_spatialexperiment", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3357,7 +3349,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/feature_annotation/spatial_autocorr/.config.vsh.yaml b/target/nextflow/feature_annotation/spatial_autocorr/.config.vsh.yaml new file mode 100644 index 0000000..b686ab0 --- /dev/null +++ b/target/nextflow/feature_annotation/spatial_autocorr/.config.vsh.yaml @@ -0,0 +1,309 @@ +name: "spatial_autocorr" +namespace: "feature_annotation" +version: "niche-compass" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "Input h5mu file." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "Modality to use." + info: null + default: + - "rna" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--layer" + description: "Layer in the AnnData object to use.\n- If `attr` is 'X', this is\ + \ the key in `.layers` to use. If not provided, `.X` is used.\n- If `attr` is\ + \ 'obsm', this is the key in `.obsm` to use.\n" + info: null + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_genes" + description: "The features to calculate autocorrelation for. Meaning depends on\ + \ `--attr`:\n- If `attr` is 'X' (default), this must be a list of gene names\ + \ from `.var_names`. \n If not provided, it defaults to using genes in `.var['highly_variable']`\ + \ (if present), or all genes.\n This behavior can be overridden by setting\ + \ `--use_all_genes` to `true`.\n Note: You cannot pass a column name from `.var`\ + \ here.\n- If `attr` is 'obs', this must be a list of column names from `.obs`.\ + \ \n- If `attr` is 'obsm', this must be indices in `.obsm[layer]` (as strings).\n" + info: null + required: false + direction: "input" + multiple: true + multiple_sep: "," + - type: "string" + name: "--obsp_neighborhood_graph" + description: "Key in .obsp where spatial connectivities are stored." + info: null + default: + - "spatial_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--use_all_genes" + description: "Whether to use all genes even if highly variable genes are present\ + \ in .var.\nIf set to true, all genes will be used.\n" + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "string" + name: "--mode" + description: "Mode of spatial autocorrelation." + info: null + default: + - "moran" + required: false + choices: + - "moran" + - "geary" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--attr" + description: "The attribute of the AnnData object to use for calculation.\n- 'X':\ + \ Use gene expression data (default).\n- 'obs': Use cell metadata.\n- 'obsm':\ + \ Use multidimensional embeddings.\n" + info: null + default: + - "X" + required: false + choices: + - "X" + - "obs" + - "obsm" + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--n_perms" + description: "Number of permutations for p-value calculation." + info: null + default: + - 100 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--use_raw" + description: "Whether to use .raw attribute of AnnData." + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output h5mu file with results in .uns." + info: null + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +label: "Spatial Autocorrelation" +summary: "Calculate spatial autocorrelation for genes using Moran's I or Geary's C." +description: "Calculate spatial autocorrelation for genes using Moran's I or Geary's\ + \ C.\nThis allows identifying spatially variable genes.\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.neighbors.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "midcpu" + - "midmem" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.13-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + - "scanpy~=1.10.4" + - "scanpy~=1.10.4" + - "squidpy~=1.8.1" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "python" + user: false + packages: + - "pytest" + - "viashpy" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +- type: "native" + id: "native" +build_info: + config: "src/feature_annotation/spatial_autocorr/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native|native" + output: "target/nextflow/feature_annotation/spatial_autocorr" + executable: "target/nextflow/feature_annotation/spatial_autocorr/main.nf" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/nextflow/feature_annotation/spatial_autocorr/main.nf b/target/nextflow/feature_annotation/spatial_autocorr/main.nf new file mode 100644 index 0000000..8381a51 --- /dev/null +++ b/target/nextflow/feature_annotation/spatial_autocorr/main.nf @@ -0,0 +1,4030 @@ +// spatial_autocorr niche-compass +// +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +// Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be. only cast if the value is a GString + if (value instanceof GString) { + value = value as String + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value !instanceof Integer) { + try { + value = value as Integer + } catch (NumberFormatException e) { + expectedClass = "Integer" + } + } + } else if (par.type == "long") { + // cast to long if need be + if (value !instanceof Long) { + try { + value = value as Long + } catch (NumberFormatException e) { + expectedClass = "Long" + } + } + } else if (par.type == "double") { + // cast to double if need be + if (value !instanceof Double) { + try { + value = value as Double + } catch (NumberFormatException e) { + expectedClass = "Double" + } + } + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } + } + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" + } + } + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } + } + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required && arg.direction == "input") { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile) + output: + tuple val(id), path{[yamlFile]} + script: + """ + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - (key, value) are the tuples that will be saved to the state.yaml file + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + return value_ + } + return [["key": plainName_, "value": outputPerFile]] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [["key": plainName_, value: value_]] + } + } + + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutputMulti = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) + + [join_id, channelId, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chJoined, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublishStates = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublishStates, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "spatial_autocorr", + "namespace" : "feature_annotation", + "version" : "niche-compass", + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "description" : "Input h5mu file.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--modality", + "description" : "Modality to use.", + "default" : [ + "rna" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--layer", + "description" : "Layer in the AnnData object to use.\n- If `attr` is 'X', this is the key in `.layers` to use. If not provided, `.X` is used.\n- If `attr` is 'obsm', this is the key in `.obsm` to use.\n", + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--input_genes", + "description" : "The features to calculate autocorrelation for. Meaning depends on `--attr`:\n- If `attr` is 'X' (default), this must be a list of gene names from `.var_names`. \n If not provided, it defaults to using genes in `.var['highly_variable']` (if present), or all genes.\n This behavior can be overridden by setting `--use_all_genes` to `true`.\n Note: You cannot pass a column name from `.var` here.\n- If `attr` is 'obs', this must be a list of column names from `.obs`. \n- If `attr` is 'obsm', this must be indices in `.obsm[layer]` (as strings).\n", + "required" : false, + "direction" : "input", + "multiple" : true, + "multiple_sep" : "," + }, + { + "type" : "string", + "name" : "--obsp_neighborhood_graph", + "description" : "Key in .obsp where spatial connectivities are stored.", + "default" : [ + "spatial_connectivities" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean", + "name" : "--use_all_genes", + "description" : "Whether to use all genes even if highly variable genes are present in .var.\nIf set to true, all genes will be used.\n", + "default" : [ + false + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Parameters", + "arguments" : [ + { + "type" : "string", + "name" : "--mode", + "description" : "Mode of spatial autocorrelation.", + "default" : [ + "moran" + ], + "required" : false, + "choices" : [ + "moran", + "geary" + ], + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--attr", + "description" : "The attribute of the AnnData object to use for calculation.\n- 'X': Use gene expression data (default).\n- 'obs': Use cell metadata.\n- 'obsm': Use multidimensional embeddings.\n", + "default" : [ + "X" + ], + "required" : false, + "choices" : [ + "X", + "obs", + "obsm" + ], + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--n_perms", + "description" : "Number of permutations for p-value calculation.", + "default" : [ + 100 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean", + "name" : "--use_raw", + "description" : "Whether to use .raw attribute of AnnData.", + "default" : [ + false + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "Output h5mu file with results in .uns.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "python_script", + "path" : "script.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/workflows/utils/labels.config", + "dest" : "nextflow_labels.config" + } + ], + "label" : "Spatial Autocorrelation", + "summary" : "Calculate spatial autocorrelation for genes using Moran's I or Geary's C.", + "description" : "Calculate spatial autocorrelation for genes using Moran's I or Geary's C.\nThis allows identifying spatially variable genes.\n", + "test_resources" : [ + { + "type" : "python_script", + "path" : "test.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/resources_test/xenium/xenium_tiny.qc.neighbors.h5mu" + } + ], + "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "label" : [ + "midcpu", + "midmem" + ], + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "python:3.13-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "niche-compass", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "procps" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "anndata~=0.12.7", + "awkward", + "mudata~=0.3.2", + "scanpy~=1.10.4", + "scanpy~=1.10.4", + "squidpy~=1.8.1" + ], + "script" : [ + "exec(\\"try:\\\\n import zarr; from importlib.metadata import version\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: assert int(version(\\\\\\"zarr\\\\\\").partition(\\\\\\".\\\\\\")[0]) > 2\\")" + ], + "upgrade" : true + } + ], + "test_setup" : [ + { + "type" : "python", + "user" : false, + "packages" : [ + "pytest", + "viashpy" + ], + "upgrade" : true + } + ] + }, + { + "type" : "native", + "id" : "native" + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/feature_annotation/spatial_autocorr/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native|native", + "output" : "/workdir/root/repo/target/nextflow/feature_annotation/spatial_autocorr", + "viash_version" : "0.9.4", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", + "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" + }, + "package_config" : { + "name" : "openpipeline_spatial", + "version" : "niche-compass", + "info" : { + "test_resources" : [ + { + "type" : "s3", + "path" : "s3://openpipelines-bio/openpipeline_spatial/resources_test", + "dest" : "resources_test" + } + ] + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "viash_version" : "0.9.4", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", + "config_mods" : [ + ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'niche-compass'" + ], + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.py" +cat > "$tempscript" << VIASHMAIN +import mudata as mu +import squidpy as sq + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'layer': $( if [ ! -z ${VIASH_PAR_LAYER+x} ]; then echo "r'${VIASH_PAR_LAYER//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'input_genes': $( if [ ! -z ${VIASH_PAR_INPUT_GENES+x} ]; then echo "r'${VIASH_PAR_INPUT_GENES//\\'/\\'\\"\\'\\"r\\'}'.split(',')"; else echo None; fi ), + 'obsp_neighborhood_graph': $( if [ ! -z ${VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH+x} ]; then echo "r'${VIASH_PAR_OBSP_NEIGHBORHOOD_GRAPH//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'use_all_genes': $( if [ ! -z ${VIASH_PAR_USE_ALL_GENES+x} ]; then echo "r'${VIASH_PAR_USE_ALL_GENES//\\'/\\'\\"\\'\\"r\\'}'.lower() == 'true'"; else echo None; fi ), + 'mode': $( if [ ! -z ${VIASH_PAR_MODE+x} ]; then echo "r'${VIASH_PAR_MODE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'attr': $( if [ ! -z ${VIASH_PAR_ATTR+x} ]; then echo "r'${VIASH_PAR_ATTR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'n_perms': $( if [ ! -z ${VIASH_PAR_N_PERMS+x} ]; then echo "int(r'${VIASH_PAR_N_PERMS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'use_raw': $( if [ ! -z ${VIASH_PAR_USE_RAW+x} ]; then echo "r'${VIASH_PAR_USE_RAW//\\'/\\'\\"\\'\\"r\\'}'.lower() == 'true'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + + +def main(): + print("Reading input MuData...", flush=True) + mdata = mu.read_h5mu(par["input"]) + adata = mdata.mod[par["modality"]] + + # Check for connectivity key + if par["obsp_neighborhood_graph"] not in adata.obsp: + raise ValueError( + f"Connectivity key '{par['obsp_neighborhood_graph']}' not found in .obsp of modality '{par['modality']}'." + ) + + genes = par["input_genes"] + if genes and len(genes) == 0: + genes = None + + if genes is None and par["use_all_genes"] and par["attr"] == "X": + genes = list(adata.var_names) + + # Handle layer + layer = par["layer"] + if layer == "None": + layer = None + + print(f"Calculating spatial autocorrelation ({par['mode']})...", flush=True) + + # Run Squidpy spatial_autocorr + # Note: sq.gr.spatial_autocorr modifies adata in-place, adding results to .uns + sq.gr.spatial_autocorr( + adata, + connectivity_key=par["obsp_neighborhood_graph"], + genes=genes, + mode=par["mode"], + attr=par["attr"], + n_perms=par["n_perms"], + layer=layer, + use_raw=par["use_raw"], + ) + + result_key = f"{par['mode']}I" if par["mode"] == "moran" else f"{par['mode']}C" + + if result_key in adata.uns: + # Log top spatially variable genes + df = adata.uns[result_key] + if not df.empty: + sort_col = "I" if par["mode"] == "moran" else "C" + print("Top spatially variable genes:", flush=True) + print(df.sort_values(by=sort_col, ascending=False).head(), flush=True) + else: + print( + f"Warning: Expected key '{result_key}' not found in .uns after calculation.", + flush=True, + ) + + print("Writing output...", flush=True) + mdata.write_h5mu(par["output"]) + print("Done!", flush=True) + + +if __name__ == "__main__": + main() +VIASHMAIN +python -B "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val + .replaceAll('\\$id', id) + .replaceAll('\\$\\{id\\}', id) + .replaceAll('\\$key', key) + .replaceAll('\\$\\{key\\}', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = _getScriptLoader(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline_spatial/feature_annotation/spatial_autocorr", + "tag" : "niche-compass" + }, + "label" : [ + "midcpu", + "midmem" + ], + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/feature_annotation/spatial_autocorr/nextflow.config b/target/nextflow/feature_annotation/spatial_autocorr/nextflow.config new file mode 100644 index 0000000..234dcbb --- /dev/null +++ b/target/nextflow/feature_annotation/spatial_autocorr/nextflow.config @@ -0,0 +1,125 @@ +manifest { + name = 'feature_annotation/spatial_autocorr' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'niche-compass' + description = 'Calculate spatial autocorrelation for genes using Moran\'s I or Geary\'s C.\nThis allows identifying spatially variable genes.\n' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/feature_annotation/spatial_autocorr/nextflow_labels.config b/target/nextflow/feature_annotation/spatial_autocorr/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/nextflow/feature_annotation/spatial_autocorr/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/nextflow/feature_annotation/spatial_autocorr/nextflow_schema.json b/target/nextflow/feature_annotation/spatial_autocorr/nextflow_schema.json new file mode 100644 index 0000000..074260c --- /dev/null +++ b/target/nextflow/feature_annotation/spatial_autocorr/nextflow_schema.json @@ -0,0 +1,133 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "spatial_autocorr", + "description": "Calculate spatial autocorrelation for genes using Moran's I or Geary's C.\nThis allows identifying spatially variable genes.\n", + "type": "object", + "$defs": { + "inputs": { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + "input": { + "type": "string", + "format": "path", + "exists": true, + "description": "Input h5mu file.", + "help_text": "Type: `file`, multiple: `False`, required, direction: `input`. " + }, + "modality": { + "type": "string", + "description": "Modality to use.", + "help_text": "Type: `string`, multiple: `False`, default: `\"rna\"`. ", + "default": "rna" + }, + "layer": { + "type": "string", + "description": "Layer in the AnnData object to use.\n- If `attr` is 'X', this is the key in `.layers` to use", + "help_text": "Type: `string`, multiple: `False`. " + }, + "input_genes": { + "type": "array", + "items": { + "type": "string" + }, + "description": "The features to calculate autocorrelation for", + "help_text": "Type: `string`, multiple: `True`. " + }, + "obsp_neighborhood_graph": { + "type": "string", + "description": "Key in .obsp where spatial connectivities are stored.", + "help_text": "Type: `string`, multiple: `False`, default: `\"spatial_connectivities\"`. ", + "default": "spatial_connectivities" + }, + "use_all_genes": { + "type": "boolean", + "description": "Whether to use all genes even if highly variable genes are present in .var.\nIf set to true, all genes will be used.\n", + "help_text": "Type: `boolean`, multiple: `False`, default: `false`. ", + "default": false + } + } + }, + "outputs": { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + "output": { + "type": "string", + "format": "path", + "description": "Output h5mu file with results in .uns.", + "help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output\"`, direction: `output`. ", + "default": "$id.$key.output" + } + } + }, + "parameters": { + "title": "Parameters", + "type": "object", + "description": "No description", + "properties": { + "mode": { + "type": "string", + "description": "Mode of spatial autocorrelation.", + "help_text": "Type: `string`, multiple: `False`, default: `\"moran\"`, choices: ``moran`, `geary``. ", + "enum": [ + "moran", + "geary" + ], + "default": "moran" + }, + "attr": { + "type": "string", + "description": "The attribute of the AnnData object to use for calculation.\n- 'X': Use gene expression data (default).\n- 'obs': Use cell metadata.\n- 'obsm': Use multidimensional embeddings.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"X\"`, choices: ``X`, `obs`, `obsm``. ", + "enum": [ + "X", + "obs", + "obsm" + ], + "default": "X" + }, + "n_perms": { + "type": "integer", + "description": "Number of permutations for p-value calculation.", + "help_text": "Type: `integer`, multiple: `False`, default: `100`. ", + "default": 100 + }, + "use_raw": { + "type": "boolean", + "description": "Whether to use .raw attribute of AnnData.", + "help_text": "Type: `boolean`, multiple: `False`, default: `false`. ", + "default": false + } + } + }, + "nextflow input-output arguments": { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + "publish_dir": { + "type": "string", + "description": "Path to an output directory.", + "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/inputs" + }, + { + "$ref": "#/$defs/outputs" + }, + { + "$ref": "#/$defs/parameters" + }, + { + "$ref": "#/$defs/nextflow input-output arguments" + } + ] +} diff --git a/target/nextflow/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml b/target/nextflow/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml new file mode 100644 index 0000000..054323a --- /dev/null +++ b/target/nextflow/feature_annotation/xenium_spatial_statistics/.config.vsh.yaml @@ -0,0 +1,279 @@ +name: "xenium_spatial_statistics" +namespace: "feature_annotation" +version: "niche-compass" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "A MuData file containing spatial transcriptomics data and a\npre-computed\ + \ spatial neighborhood graph in `.obsp`.\n" + info: null + example: + - "input.h5mu" + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "The name of the modality to use from the MuData object.\nThis specifies\ + \ which AnnData object contains the spatial data.\n" + info: null + default: + - "rna" + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--obsm_spatial_coordinates" + description: "The key in `.obsm` where spatial coordinates are stored.\nExpected\ + \ shape is (n_cells, 2) for 2D coordinates.\n" + info: null + default: + - "spatial" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "The output MuData file with calculated spatial statistics.\nPer-cell\ + \ metrics are stored in `.obs` and global statistics in `.uns`.\n" + info: null + default: + - "output.h5mu" + must_exist: true + create_parent: true + required: false + direction: "output" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_prefix" + description: "Prefix to add to all generated column names in `.obs`.\n" + info: null + default: + - "spatial_" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "double" + name: "--density_bandwidth" + description: "Bandwidth parameter for Gaussian kernel density estimation in spatial\ + \ units\n(typically microns). Larger values create smoother density estimates.\n" + info: null + default: + - 50.0 + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "boolean" + name: "--calculate_ripley_l" + description: "Whether to calculate Ripley's L statistic. \nWarning: This is an\ + \ O(N^2) operation and can be very slow for large datasets (>5000 cells).\n" + info: null + default: + - false + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "integer" + name: "--n_subsample_ripley" + description: "Number of cells to subsample for Ripley's L calculation. \nIf -1,\ + \ use all cells. Recommended to keep this below 5000 for performance.\n" + info: null + default: + - -1 + required: false + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +label: "Xenium Spatial Statistics" +description: "Calculate various spatial statistics from Xenium data including cell\ + \ morphology\nratios, position-based features, local density metrics, and global\ + \ spatial patterns.\nThis component expects a MuData object with a pre-computed\ + \ spatial neighborhood graph\nin `.obsp`.\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.neighbors.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "singlecpu" + - "midmem" + - "lowdisk" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.13-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + - "scanpy~=1.10.4" + - "scanpy~=1.10.4" + - "squidpy~=1.8.1" + - "scikit-learn" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "python" + user: false + packages: + - "pytest" + - "viashpy" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +- type: "native" + id: "native" +build_info: + config: "src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native|native" + output: "target/nextflow/feature_annotation/xenium_spatial_statistics" + executable: "target/nextflow/feature_annotation/xenium_spatial_statistics/main.nf" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/nextflow/feature_annotation/xenium_spatial_statistics/main.nf b/target/nextflow/feature_annotation/xenium_spatial_statistics/main.nf new file mode 100644 index 0000000..800bfab --- /dev/null +++ b/target/nextflow/feature_annotation/xenium_spatial_statistics/main.nf @@ -0,0 +1,4234 @@ +// xenium_spatial_statistics niche-compass +// +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +// Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be. only cast if the value is a GString + if (value instanceof GString) { + value = value as String + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value !instanceof Integer) { + try { + value = value as Integer + } catch (NumberFormatException e) { + expectedClass = "Integer" + } + } + } else if (par.type == "long") { + // cast to long if need be + if (value !instanceof Long) { + try { + value = value as Long + } catch (NumberFormatException e) { + expectedClass = "Long" + } + } + } else if (par.type == "double") { + // cast to double if need be + if (value !instanceof Double) { + try { + value = value as Double + } catch (NumberFormatException e) { + expectedClass = "Double" + } + } + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } + } + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" + } + } + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } + } + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required && arg.direction == "input") { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile) + output: + tuple val(id), path{[yamlFile]} + script: + """ + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - (key, value) are the tuples that will be saved to the state.yaml file + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + return value_ + } + return [["key": plainName_, "value": outputPerFile]] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [["key": plainName_, value: value_]] + } + } + + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutputMulti = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) + + [join_id, channelId, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chJoined, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublishStates = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublishStates, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "xenium_spatial_statistics", + "namespace" : "feature_annotation", + "version" : "niche-compass", + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "description" : "A MuData file containing spatial transcriptomics data and a\npre-computed spatial neighborhood graph in `.obsp`.\n", + "example" : [ + "input.h5mu" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--modality", + "description" : "The name of the modality to use from the MuData object.\nThis specifies which AnnData object contains the spatial data.\n", + "default" : [ + "rna" + ], + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--obsm_spatial_coordinates", + "description" : "The key in `.obsm` where spatial coordinates are stored.\nExpected shape is (n_cells, 2) for 2D coordinates.\n", + "default" : [ + "spatial" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "The output MuData file with calculated spatial statistics.\nPer-cell metrics are stored in `.obs` and global statistics in `.uns`.\n", + "default" : [ + "output.h5mu" + ], + "must_exist" : true, + "create_parent" : true, + "required" : false, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--output_prefix", + "description" : "Prefix to add to all generated column names in `.obs`.\n", + "default" : [ + "spatial_" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Parameters", + "arguments" : [ + { + "type" : "double", + "name" : "--density_bandwidth", + "description" : "Bandwidth parameter for Gaussian kernel density estimation in spatial units\n(typically microns). Larger values create smoother density estimates.\n", + "default" : [ + 50.0 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "boolean", + "name" : "--calculate_ripley_l", + "description" : "Whether to calculate Ripley's L statistic. \nWarning: This is an O(N^2) operation and can be very slow for large datasets (>5000 cells).\n", + "default" : [ + false + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "integer", + "name" : "--n_subsample_ripley", + "description" : "Number of cells to subsample for Ripley's L calculation. \nIf -1, use all cells. Recommended to keep this below 5000 for performance.\n", + "default" : [ + -1 + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "python_script", + "path" : "script.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/workflows/utils/labels.config", + "dest" : "nextflow_labels.config" + } + ], + "label" : "Xenium Spatial Statistics", + "description" : "Calculate various spatial statistics from Xenium data including cell morphology\nratios, position-based features, local density metrics, and global spatial patterns.\nThis component expects a MuData object with a pre-computed spatial neighborhood graph\nin `.obsp`.\n", + "test_resources" : [ + { + "type" : "python_script", + "path" : "test.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/resources_test/xenium/xenium_tiny.qc.neighbors.h5mu" + } + ], + "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "label" : [ + "singlecpu", + "midmem", + "lowdisk" + ], + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "python:3.13-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "niche-compass", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "procps" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "anndata~=0.12.7", + "awkward", + "mudata~=0.3.2", + "scanpy~=1.10.4", + "scanpy~=1.10.4", + "squidpy~=1.8.1", + "scikit-learn" + ], + "script" : [ + "exec(\\"try:\\\\n import zarr; from importlib.metadata import version\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: assert int(version(\\\\\\"zarr\\\\\\").partition(\\\\\\".\\\\\\")[0]) > 2\\")" + ], + "upgrade" : true + } + ], + "test_setup" : [ + { + "type" : "python", + "user" : false, + "packages" : [ + "pytest", + "viashpy" + ], + "upgrade" : true + } + ] + }, + { + "type" : "native", + "id" : "native" + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/feature_annotation/xenium_spatial_statistics/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native|native", + "output" : "/workdir/root/repo/target/nextflow/feature_annotation/xenium_spatial_statistics", + "viash_version" : "0.9.4", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", + "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" + }, + "package_config" : { + "name" : "openpipeline_spatial", + "version" : "niche-compass", + "info" : { + "test_resources" : [ + { + "type" : "s3", + "path" : "s3://openpipelines-bio/openpipeline_spatial/resources_test", + "dest" : "resources_test" + } + ] + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "viash_version" : "0.9.4", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", + "config_mods" : [ + ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'niche-compass'" + ], + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.py" +cat > "$tempscript" << VIASHMAIN +import sys +import warnings + +import mudata as mu +import numpy as np +import squidpy as sq +from scipy.spatial import ConvexHull, Voronoi, distance_matrix +from sklearn.neighbors import KernelDensity + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'obsm_spatial_coordinates': $( if [ ! -z ${VIASH_PAR_OBSM_SPATIAL_COORDINATES+x} ]; then echo "r'${VIASH_PAR_OBSM_SPATIAL_COORDINATES//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'output_prefix': $( if [ ! -z ${VIASH_PAR_OUTPUT_PREFIX+x} ]; then echo "r'${VIASH_PAR_OUTPUT_PREFIX//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'density_bandwidth': $( if [ ! -z ${VIASH_PAR_DENSITY_BANDWIDTH+x} ]; then echo "float(r'${VIASH_PAR_DENSITY_BANDWIDTH//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'calculate_ripley_l': $( if [ ! -z ${VIASH_PAR_CALCULATE_RIPLEY_L+x} ]; then echo "r'${VIASH_PAR_CALCULATE_RIPLEY_L//\\'/\\'\\"\\'\\"r\\'}'.lower() == 'true'"; else echo None; fi ), + 'n_subsample_ripley': $( if [ ! -z ${VIASH_PAR_N_SUBSAMPLE_RIPLEY+x} ]; then echo "int(r'${VIASH_PAR_N_SUBSAMPLE_RIPLEY//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + + +def calculate_morphology_metrics(adata, prefix): + """Calculate cell morphology ratios and features.""" + print(" Calculating morphology metrics...", flush=True) + + if "cell_area" not in adata.obs or "nucleus_area" not in adata.obs: + warnings.warn( + "cell_area and/or nucleus_area not found in .obs. " + "Skipping morphology metrics." + ) + return + + # Nucleus to cell area ratio - can be removed once https://github.com/openpipelines-bio/openpipeline_qc/issues/18 is fixed. + adata.obs[f"{prefix}nucleus_cell_ratio"] = ( + adata.obs["nucleus_area"] / adata.obs["cell_area"] + ) + + # Cell area percentile (relative size) + adata.obs[f"{prefix}cell_area_percentile"] = ( + adata.obs["cell_area"].rank(pct=True) * 100 + ) + + print( + f" Added: {prefix}nucleus_cell_ratio, {prefix}cell_area_percentile", + flush=True, + ) + + +def calculate_position_features(adata, spatial_coords, prefix): + """Calculate position-based features.""" + print(" Calculating position-based features...", flush=True) + + # Tissue centroid + centroid = spatial_coords.mean(axis=0) + + # Distance to centroid + distances_to_centroid = np.linalg.norm(spatial_coords - centroid, axis=1) + adata.obs[f"{prefix}distance_to_centroid"] = distances_to_centroid + + # Normalized coordinates (0-1 scale) + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + coord_range = max_coords - min_coords + + normalized_coords = (spatial_coords - min_coords) / coord_range + adata.obs[f"{prefix}norm_x"] = normalized_coords[:, 0] + adata.obs[f"{prefix}norm_y"] = normalized_coords[:, 1] + + # Distance to convex hull boundary + try: + hull = ConvexHull(spatial_coords) + hull_points = spatial_coords[hull.vertices] + + # For each point, find distance to nearest hull vertex (approximation) + distances_to_boundary = np.min( + distance_matrix(spatial_coords, hull_points), axis=1 + ) + adata.obs[f"{prefix}distance_to_boundary"] = distances_to_boundary + print( + " Added: distance_to_centroid, norm_x, norm_y, distance_to_boundary", + flush=True, + ) + except Exception as e: + warnings.warn(f"Could not calculate convex hull: {e}") + print(" Added: distance_to_centroid, norm_x, norm_y", flush=True) + + +def calculate_density_metrics(adata, spatial_coords, bandwidth, prefix): + """Calculate local density metrics.""" + print(" Calculating density metrics...", flush=True) + + # Kernel density estimation + kde = KernelDensity(bandwidth=bandwidth, kernel="gaussian") + kde.fit(spatial_coords) + log_density = kde.score_samples(spatial_coords) + adata.obs[f"{prefix}kernel_density"] = np.exp(log_density) + + # Calculate degree centrality (as a proxy for local density / number of neighbors) + # This assumes spatial_connectivities is already present in .obsp (from build_spatial_graph) + if "spatial_connectivities" in adata.obsp: + spatial_connectivities = adata.obsp["spatial_connectivities"] + degrees = spatial_connectivities.sum(axis=1) + # If sparse matrix, it returns matrix object, need to convert to array + if hasattr(degrees, "A1"): + degrees = degrees.A1 + adata.obs[f"{prefix}graph_degree"] = degrees + + print(" Added: kernel_density, graph_degree", flush=True) + else: + print( + " Warning: 'spatial_connectivities' not found in .obsp. Skipping graph_degree.", + flush=True, + ) + print(" Added: kernel_density", flush=True) + + +def calculate_voronoi_metrics(adata, spatial_coords, prefix): + """Calculate Voronoi tessellation statistics.""" + print(" Calculating Voronoi tessellation...", flush=True) + + try: + vor = Voronoi(spatial_coords) + + polygon_areas = [] + neighbor_counts = [] + + for point_idx in range(len(spatial_coords)): + region_idx = vor.point_region[point_idx] + region = vor.regions[region_idx] + + # Skip infinite regions + if -1 in region or len(region) == 0: + polygon_areas.append(np.nan) + neighbor_counts.append(np.nan) + continue + + # Calculate polygon area using shoelace formula + vertices = vor.vertices[region] + x = vertices[:, 0] + y = vertices[:, 1] + area = 0.5 * np.abs(np.dot(x, np.roll(y, 1)) - np.dot(y, np.roll(x, 1))) + polygon_areas.append(area) + + # Count neighbors (shared vertices) + neighbor_counts.append(len(region)) + + adata.obs[f"{prefix}voronoi_area"] = polygon_areas + adata.obs[f"{prefix}voronoi_neighbors"] = neighbor_counts + + print(" Added: voronoi_area, voronoi_neighbors", flush=True) + except Exception as e: + warnings.warn(f"Could not calculate Voronoi tessellation: {e}") + + +def calculate_global_statistics(adata, spatial_coords, par): + """Calculate global spatial pattern statistics.""" + print(" Calculating global spatial statistics...", flush=True) + + stats = {} + + # Global density + try: + hull = ConvexHull(spatial_coords) + area = hull.volume # In 2D, volume of convex hull is the area + stats["area_calculation_method"] = "convex_hull" + stats["cell_density"] = len(spatial_coords) / area + stats["total_area"] = area + stats["n_cells"] = len(spatial_coords) + except Exception as e: + print(f" Error: Could not calculate convex hull area ({e})", flush=True) + + # Ripley's L (using squidpy) + if par["calculate_ripley_l"]: + print(" Computing Ripley's L function (Squidpy)...", flush=True) + + n_subsample = par["n_subsample_ripley"] + + try: + # Create a working AnnData for Ripley's + if n_subsample > 0 and len(adata) > n_subsample: + print(f" Subsampling to {n_subsample} random cells...", flush=True) + + # Use numpy to generate random indices + indices = np.random.choice(len(adata), n_subsample, replace=False) + adata_ripley = adata[indices].copy() + stats["ripley_l_subsampled"] = True + else: + adata_ripley = adata.copy() + stats["ripley_l_subsampled"] = False + + # Squidpy requires a cluster key. We create a dummy one for "global" context. + adata_ripley.obs["_temp_global"] = "all" + adata_ripley.obs["_temp_global"] = adata_ripley.obs["_temp_global"].astype( + "category" + ) + + # Calculate Ripley's L statistic + # This stores the result in adata.uns['all_L'] + sq.gr.ripley( + adata_ripley, + cluster_key="_temp_global", + mode="L", + spatial_key=par["obsm_spatial_coordinates"], + n_simulations=50, + ) + + # Extract basic stats from the results + if "all_L" in adata_ripley.uns and "L_stat" in adata_ripley.uns["all_L"]: + l_stat_df = adata_ripley.uns["all_L"]["L_stat"] + stats["ripley_l_max"] = float(l_stat_df.max().max()) + stats["ripley_l_mean"] = float(l_stat_df.mean().mean()) + + # Clean up + if "all_L" in adata_ripley.uns: + del adata_ripley.uns["all_L"] + + except Exception as e: + print(f" Error calculating Ripley's L: {e}", flush=True) + import traceback + + traceback.print_exc() + else: + print(" Skipping Ripley's L (disabled in config)...", flush=True) + + # Spatial extent + min_coords = spatial_coords.min(axis=0) + max_coords = spatial_coords.max(axis=0) + + stats["spatial_extent_x"] = float(max_coords[0] - min_coords[0]) + stats["spatial_extent_y"] = float(max_coords[1] - min_coords[1]) + stats["centroid_x"] = float(spatial_coords[:, 0].mean()) + stats["centroid_y"] = float(spatial_coords[:, 1].mean()) + + if "cell_density" in stats: + print( + f" Global stats: cell_density={stats['cell_density']:.4f}", + flush=True, + ) + + return stats + + +def main(par): + print(f"\\\\n>>> Reading MuData from '{par['input']}'...", flush=True) + mdata = mu.read_h5mu(par["input"]) + print(mdata, flush=True) + + print(f"\\\\n>>> Extracting modality '{par['modality']}'...", flush=True) + if par["modality"] not in mdata.mod: + raise KeyError( + f"Modality '{par['modality']}' not found in MuData. " + f"Available modalities: {list(mdata.mod.keys())}" + ) + adata = mdata[par["modality"]] + print(adata, flush=True) + + print( + f"\\\\n>>> Extracting spatial coordinates from .obsm['{par['obsm_spatial_coordinates']}']...", + flush=True, + ) + if par["obsm_spatial_coordinates"] not in adata.obsm: + raise KeyError( + f"Spatial key '{par['obsm_spatial_coordinates']}' not found in .obsm. " + f"Available keys: {list(adata.obsm.keys())}" + ) + + spatial_coords = adata.obsm[par["obsm_spatial_coordinates"]] + if spatial_coords.shape[1] != 2: + raise ValueError( + f"Expected 2D spatial coordinates, got shape {spatial_coords.shape}" + ) + print(f" Shape: {spatial_coords.shape} (n_cells × 2)", flush=True) + + prefix = par["output_prefix"] + + # Calculate morphology metrics + print("\\\\n>>> Calculating morphology metrics...", flush=True) + calculate_morphology_metrics(adata, prefix) + + # Calculate position features + print("\\\\n>>> Calculating position-based features...", flush=True) + calculate_position_features(adata, spatial_coords, prefix) + + # Calculate density metrics + print("\\\\n>>> Calculating density metrics...", flush=True) + calculate_density_metrics( + adata, + spatial_coords, + par["density_bandwidth"], + prefix, + ) + + # Calculate Voronoi tessellation + print("\\\\n>>> Calculating Voronoi tessellation...", flush=True) + calculate_voronoi_metrics(adata, spatial_coords, prefix) + + # Calculate global statistics + print("\\\\n>>> Calculating global spatial statistics...", flush=True) + global_stats = calculate_global_statistics(adata, spatial_coords, par) + + # Store in uns + if "spatial_stats" not in adata.uns: + adata.uns["spatial_stats"] = {} + adata.uns["spatial_stats"].update(global_stats) + + print(f"\\\\n>>> Writing output to '{par['output']}'...", flush=True) + mdata.write_h5mu(par["output"]) + + print("\\\\n>>> Done!\\\\n", flush=True) + + +if __name__ == "__main__": + sys.exit(main(par)) +VIASHMAIN +python -B "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val + .replaceAll('\\$id', id) + .replaceAll('\\$\\{id\\}', id) + .replaceAll('\\$key', key) + .replaceAll('\\$\\{key\\}', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = _getScriptLoader(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline_spatial/feature_annotation/xenium_spatial_statistics", + "tag" : "niche-compass" + }, + "label" : [ + "singlecpu", + "midmem", + "lowdisk" + ], + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow.config b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow.config new file mode 100644 index 0000000..a24db83 --- /dev/null +++ b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow.config @@ -0,0 +1,125 @@ +manifest { + name = 'feature_annotation/xenium_spatial_statistics' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'niche-compass' + description = 'Calculate various spatial statistics from Xenium data including cell morphology\nratios, position-based features, local density metrics, and global spatial patterns.\nThis component expects a MuData object with a pre-computed spatial neighborhood graph\nin `.obsp`.\n' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_labels.config b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_schema.json b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_schema.json new file mode 100644 index 0000000..493a00b --- /dev/null +++ b/target/nextflow/feature_annotation/xenium_spatial_statistics/nextflow_schema.json @@ -0,0 +1,105 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "xenium_spatial_statistics", + "description": "Calculate various spatial statistics from Xenium data including cell morphology\nratios, position-based features, local density metrics, and global spatial patterns.\nThis component expects a MuData object with a pre-computed spatial neighborhood graph\nin `.obsp`.\n", + "type": "object", + "$defs": { + "inputs": { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + "input": { + "type": "string", + "format": "path", + "exists": true, + "description": "A MuData file containing spatial transcriptomics data and a\npre-computed spatial neighborhood graph in `.obsp`.\n", + "help_text": "Type: `file`, multiple: `False`, required, direction: `input`, example: `\"input.h5mu\"`. " + }, + "modality": { + "type": "string", + "description": "The name of the modality to use from the MuData object.\nThis specifies which AnnData object contains the spatial data.\n", + "help_text": "Type: `string`, multiple: `False`, required, default: `\"rna\"`. ", + "default": "rna" + }, + "obsm_spatial_coordinates": { + "type": "string", + "description": "The key in `.obsm` where spatial coordinates are stored.\nExpected shape is (n_cells, 2) for 2D coordinates.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"spatial\"`. ", + "default": "spatial" + } + } + }, + "outputs": { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + "output": { + "type": "string", + "format": "path", + "description": "The output MuData file with calculated spatial statistics.\nPer-cell metrics are stored in `.obs` and global statistics in `.uns`.\n", + "help_text": "Type: `file`, multiple: `False`, default: `\"output.h5mu\"`, direction: `output`. ", + "default": "output.h5mu" + }, + "output_prefix": { + "type": "string", + "description": "Prefix to add to all generated column names in `.obs`.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"spatial_\"`. ", + "default": "spatial_" + } + } + }, + "parameters": { + "title": "Parameters", + "type": "object", + "description": "No description", + "properties": { + "density_bandwidth": { + "type": "number", + "description": "Bandwidth parameter for Gaussian kernel density estimation in spatial units\n(typically microns)", + "help_text": "Type: `double`, multiple: `False`, default: `50.0`. ", + "default": 50.0 + }, + "calculate_ripley_l": { + "type": "boolean", + "description": "Whether to calculate Ripley's L statistic", + "help_text": "Type: `boolean`, multiple: `False`, default: `false`. ", + "default": false + }, + "n_subsample_ripley": { + "type": "integer", + "description": "Number of cells to subsample for Ripley's L calculation", + "help_text": "Type: `integer`, multiple: `False`, default: `-1`. ", + "default": -1 + } + } + }, + "nextflow input-output arguments": { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + "publish_dir": { + "type": "string", + "description": "Path to an output directory.", + "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/inputs" + }, + { + "$ref": "#/$defs/outputs" + }, + { + "$ref": "#/$defs/parameters" + }, + { + "$ref": "#/$defs/nextflow input-output arguments" + } + ] +} diff --git a/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml b/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml index 9849c68..4ac7490 100644 --- a/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml +++ b/target/nextflow/mapping/spaceranger_count/.config.vsh.yaml @@ -330,7 +330,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" keywords: - "spaceranger" links: @@ -439,7 +439,7 @@ build_info: output: "target/nextflow/mapping/spaceranger_count" executable: "target/nextflow/mapping/spaceranger_count/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -453,7 +453,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/mapping/spaceranger_count/main.nf b/target/nextflow/mapping/spaceranger_count/main.nf index 8b206d8..da418f4 100644 --- a/target/nextflow/mapping/spaceranger_count/main.nf +++ b/target/nextflow/mapping/spaceranger_count/main.nf @@ -3433,7 +3433,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "keywords" : [ @@ -3569,7 +3569,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/mapping/spaceranger_count", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3589,7 +3589,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/neighbors/join_graphs/.config.vsh.yaml b/target/nextflow/neighbors/join_graphs/.config.vsh.yaml new file mode 100644 index 0000000..ad80ba7 --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/.config.vsh.yaml @@ -0,0 +1,295 @@ +name: "join_graphs" +namespace: "neighbors" +version: "niche-compass" +authors: +- name: "Jakub Majercik" + roles: + - "maintainer" + info: + role: "Contributor" + links: + email: "jakub@data-intuitive.com" + github: "jakubmajercik" + linkedin: "jakubmajercik" + organizations: + - name: "Data Intuitive" + href: "https://www.data-intuitive.com" + role: "Bioinformatics Engineer" +argument_groups: +- name: "Inputs" + arguments: + - type: "file" + name: "--input" + description: "Input H5MU file with pre-computed expression and spatial neighborhood\ + \ graphs." + info: null + must_exist: true + create_parent: true + required: true + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--modality" + description: "Modality from the input MuData file to process." + info: null + default: + - "rna" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_obsp_expression_graph" + description: "Key in `adata.obsp` containing the expression connectivity matrix.\n" + info: null + default: + - "connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--input_obsp_spatial_graph" + description: "Key in `adata.obsp` containing the spatial connectivity matrix.\n" + info: null + default: + - "spatial_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Parameters" + arguments: + - type: "double" + name: "--alpha" + description: "Weight of the spatial graph in the connection fusion. \nA value\ + \ of 0 results in the original expression graph; \na value of 1 results in the\ + \ spatial graph only.\n" + info: null + default: + - 0.5 + required: false + min: 0.0 + max: 1.0 + direction: "input" + multiple: false + multiple_sep: ";" +- name: "Outputs" + arguments: + - type: "file" + name: "--output" + description: "Output H5MU file path." + info: null + example: + - "output.h5mu" + must_exist: true + create_parent: true + required: true + direction: "output" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_obsp_graph" + description: "Key under which to store the fused connectivity matrix in `adata.obsp`.\n" + info: null + default: + - "spatial_expression_connectivities" + required: false + direction: "input" + multiple: false + multiple_sep: ";" + - type: "string" + name: "--output_compression" + description: "Compression format to use for the output AnnData and/or Mudata objects.\n\ + By default no compression is applied.\n" + info: null + example: + - "gzip" + required: false + choices: + - "gzip" + - "lzf" + direction: "input" + multiple: false + multiple_sep: ";" +resources: +- type: "python_script" + path: "script.py" + is_executable: true +- type: "file" + path: "setup_logger.py" +- type: "file" + path: "nextflow_labels.config" + dest: "nextflow_labels.config" +description: "Combine spatial and expression neighborhood graphs into a single graph\ + \ for \nuse in downstream clustering or trajectory analysis.\n\nThe fusion is a\ + \ linear combination of the expression connectivities \nand spatial connectivities\ + \ matrices, weighted by `alpha`.\n\n$C_{joint} = (1 - \\alpha) \\cdot C_{expression}\ + \ + \\alpha \\cdot C_{spatial}$\n" +test_resources: +- type: "python_script" + path: "test.py" + is_executable: true +- type: "file" + path: "xenium_tiny.qc.all_neighbors.pca.h5mu" +info: null +status: "enabled" +scope: + image: "public" + target: "public" +repositories: +- type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" +links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" +runners: +- type: "executable" + id: "executable" + docker_setup_strategy: "ifneedbepullelsecachedbuild" +- type: "nextflow" + id: "nextflow" + directives: + label: + - "midcpu" + - "midmem" + - "middisk" + tag: "$id" + auto: + simplifyInput: true + simplifyOutput: false + transcript: false + publish: false + config: + labels: + mem1gb: "memory = 1000000000.B" + mem2gb: "memory = 2000000000.B" + mem5gb: "memory = 5000000000.B" + mem10gb: "memory = 10000000000.B" + mem20gb: "memory = 20000000000.B" + mem50gb: "memory = 50000000000.B" + mem100gb: "memory = 100000000000.B" + mem200gb: "memory = 200000000000.B" + mem500gb: "memory = 500000000000.B" + mem1tb: "memory = 1000000000000.B" + mem2tb: "memory = 2000000000000.B" + mem5tb: "memory = 5000000000000.B" + mem10tb: "memory = 10000000000000.B" + mem20tb: "memory = 20000000000000.B" + mem50tb: "memory = 50000000000000.B" + mem100tb: "memory = 100000000000000.B" + mem200tb: "memory = 200000000000000.B" + mem500tb: "memory = 500000000000000.B" + mem1gib: "memory = 1073741824.B" + mem2gib: "memory = 2147483648.B" + mem4gib: "memory = 4294967296.B" + mem8gib: "memory = 8589934592.B" + mem16gib: "memory = 17179869184.B" + mem32gib: "memory = 34359738368.B" + mem64gib: "memory = 68719476736.B" + mem128gib: "memory = 137438953472.B" + mem256gib: "memory = 274877906944.B" + mem512gib: "memory = 549755813888.B" + mem1tib: "memory = 1099511627776.B" + mem2tib: "memory = 2199023255552.B" + mem4tib: "memory = 4398046511104.B" + mem8tib: "memory = 8796093022208.B" + mem16tib: "memory = 17592186044416.B" + mem32tib: "memory = 35184372088832.B" + mem64tib: "memory = 70368744177664.B" + mem128tib: "memory = 140737488355328.B" + mem256tib: "memory = 281474976710656.B" + mem512tib: "memory = 562949953421312.B" + cpu1: "cpus = 1" + cpu2: "cpus = 2" + cpu5: "cpus = 5" + cpu10: "cpus = 10" + cpu20: "cpus = 20" + cpu50: "cpus = 50" + cpu100: "cpus = 100" + cpu200: "cpus = 200" + cpu500: "cpus = 500" + cpu1000: "cpus = 1000" + script: + - "includeConfig(\"nextflow_labels.config\")" + debug: false + container: "docker" +engines: +- type: "docker" + id: "docker" + image: "python:3.12-slim" + target_registry: "images.viash-hub.com" + target_tag: "niche-compass" + namespace_separator: "/" + setup: + - type: "apt" + packages: + - "procps" + interactive: false + - type: "python" + user: false + packages: + - "scanpy~=1.10.4" + - "anndata~=0.12.7" + - "awkward" + - "mudata~=0.3.2" + script: + - "exec(\"try:\\n import zarr; from importlib.metadata import version\\nexcept\ + \ ModuleNotFoundError:\\n exit(0)\\nelse: assert int(version(\\\"zarr\\\"\ + ).partition(\\\".\\\")[0]) > 2\")" + upgrade: true + test_setup: + - type: "apt" + packages: + - "git" + interactive: false + - type: "python" + user: false + packages: + - "viashpy==0.9.0" + github: + - "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" + upgrade: true + entrypoint: [] + cmd: null +- type: "native" + id: "native" +build_info: + config: "src/neighbors/join_graphs/config.vsh.yaml" + runner: "nextflow" + engine: "docker|native" + output: "target/nextflow/neighbors/join_graphs" + executable: "target/nextflow/neighbors/join_graphs/main.nf" + viash_version: "0.9.4" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" + git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" +package_config: + name: "openpipeline_spatial" + version: "niche-compass" + info: + test_resources: + - type: "s3" + path: "s3://openpipelines-bio/openpipeline_spatial/resources_test" + dest: "resources_test" + repositories: + - type: "vsh" + name: "openpipeline" + repo: "openpipeline" + tag: "v4.0.3" + viash_version: "0.9.4" + source: "src" + target: "target" + config_mods: + - ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\ + .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\ + )'" + - ".engines += { type: \"native\" }" + - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'" + - ".engines[.type == 'docker'].target_tag := 'niche-compass'" + organization: "vsh" + links: + repository: "https://github.com/openpipelines-bio/openpipeline_spatial" + docker_registry: "ghcr.io" diff --git a/target/nextflow/neighbors/join_graphs/main.nf b/target/nextflow/neighbors/join_graphs/main.nf new file mode 100644 index 0000000..59f01b7 --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/main.nf @@ -0,0 +1,4012 @@ +// join_graphs niche-compass +// +// This wrapper script is auto-generated by viash 0.9.4 and is thus a derivative +// work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data +// Intuitive. +// +// The component may contain files which fall under a different license. The +// authors of this component should specify the license in the header of such +// files, or include a separate license file detailing the licenses of all included +// files. +// +// Component authors: +// * Jakub Majercik (maintainer) + +//////////////////////////// +// VDSL3 helper functions // +//////////////////////////// + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_checkArgumentType.nf' +class UnexpectedArgumentTypeException extends Exception { + String errorIdentifier + String stage + String plainName + String expectedClass + String foundClass + + // ${key ? " in module '$key'" : ""}${id ? " id '$id'" : ""} + UnexpectedArgumentTypeException(String errorIdentifier, String stage, String plainName, String expectedClass, String foundClass) { + super("Error${errorIdentifier ? " $errorIdentifier" : ""}:${stage ? " $stage" : "" } argument '${plainName}' has the wrong type. " + + "Expected type: ${expectedClass}. Found type: ${foundClass}") + this.errorIdentifier = errorIdentifier + this.stage = stage + this.plainName = plainName + this.expectedClass = expectedClass + this.foundClass = foundClass + } +} + +/** + * Checks if the given value is of the expected type. If not, an exception is thrown. + * + * @param stage The stage of the argument (input or output) + * @param par The parameter definition + * @param value The value to check + * @param errorIdentifier The identifier to use in the error message + * @return The value, if it is of the expected type + * @throws UnexpectedArgumentTypeException If the value is not of the expected type +*/ +def _checkArgumentType(String stage, Map par, Object value, String errorIdentifier) { + // expectedClass will only be != null if value is not of the expected type + def expectedClass = null + def foundClass = null + + // todo: split if need be + + if (!par.required && value == null) { + expectedClass = null + } else if (par.multiple) { + if (value !instanceof Collection) { + value = [value] + } + + // split strings + value = value.collectMany{ val -> + if (val instanceof String) { + // collect() to ensure that the result is a List and not simply an array + val.split(par.multiple_sep).collect() + } else { + [val] + } + } + + // process globs + if (par.type == "file" && par.direction == "input") { + value = value.collect{ it instanceof String ? file(it, hidden: true) : it }.flatten() + } + + // check types of elements in list + try { + value = value.collect { listVal -> + _checkArgumentType(stage, par + [multiple: false], listVal, errorIdentifier) + } + } catch (UnexpectedArgumentTypeException e) { + expectedClass = "List[${e.expectedClass}]" + foundClass = "List[${e.foundClass}]" + } + } else if (par.type == "string") { + // cast to string if need be. only cast if the value is a GString + if (value instanceof GString) { + value = value as String + } + expectedClass = value instanceof String ? null : "String" + } else if (par.type == "integer") { + // cast to integer if need be + if (value !instanceof Integer) { + try { + value = value as Integer + } catch (NumberFormatException e) { + expectedClass = "Integer" + } + } + } else if (par.type == "long") { + // cast to long if need be + if (value !instanceof Long) { + try { + value = value as Long + } catch (NumberFormatException e) { + expectedClass = "Long" + } + } + } else if (par.type == "double") { + // cast to double if need be + if (value !instanceof Double) { + try { + value = value as Double + } catch (NumberFormatException e) { + expectedClass = "Double" + } + } + } else if (par.type == "float") { + // cast to float if need be + if (value !instanceof Float) { + try { + value = value as Float + } catch (NumberFormatException e) { + expectedClass = "Float" + } + } + } else if (par.type == "boolean" | par.type == "boolean_true" | par.type == "boolean_false") { + // cast to boolean if need be + if (value !instanceof Boolean) { + try { + value = value as Boolean + } catch (Exception e) { + expectedClass = "Boolean" + } + } + } else if (par.type == "file" && (par.direction == "input" || stage == "output")) { + // cast to path if need be + if (value instanceof String) { + value = file(value, hidden: true) + } + if (value instanceof File) { + value = value.toPath() + } + expectedClass = value instanceof Path ? null : "Path" + } else if (par.type == "file" && stage == "input" && par.direction == "output") { + // cast to string if need be + if (value !instanceof String) { + try { + value = value as String + } catch (Exception e) { + expectedClass = "String" + } + } + } else { + // didn't find a match for par.type + expectedClass = par.type + } + + if (expectedClass != null) { + if (foundClass == null) { + foundClass = value.getClass().getName() + } + throw new UnexpectedArgumentTypeException(errorIdentifier, stage, par.plainName, expectedClass, foundClass) + } + + return value +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processInputValues.nf' +Map _processInputValues(Map inputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.required && arg.direction == "input") { + assert inputs.containsKey(arg.plainName) && inputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required input argument '${arg.plainName}' is missing" + } + } + + inputs = inputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid input argument" + + value = _checkArgumentType("input", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return inputs +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/arguments/_processOutputValues.nf' +Map _checkValidOutputArgument(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + outputs = outputs.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && it.direction == "output" } + assert par != null : "Error in module '${key}' id '${id}': '${name}' is not a valid output argument" + + value = _checkArgumentType("output", par, value, "in module '$key' id '$id'") + + [ name, value ] + } + } + return outputs +} + +void _checkAllRequiredOuputsPresent(Map outputs, Map config, String id, String key) { + if (!workflow.stubRun) { + config.allArguments.each { arg -> + if (arg.direction == "output" && arg.required) { + assert outputs.containsKey(arg.plainName) && outputs.get(arg.plainName) != null : + "Error in module '${key}' id '${id}': required output argument '${arg.plainName}' is missing" + } + } + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/IDChecker.nf' +class IDChecker { + final def items = [] as Set + + @groovy.transform.WithWriteLock + boolean observe(String item) { + if (items.contains(item)) { + return false + } else { + items << item + return true + } + } + + @groovy.transform.WithReadLock + boolean contains(String item) { + return items.contains(item) + } + + @groovy.transform.WithReadLock + Set getItems() { + return items.clone() + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_checkUniqueIds.nf' + +/** + * Check if the ids are unique across parameter sets + * + * @param parameterSets a list of parameter sets. + */ +private void _checkUniqueIds(List>> parameterSets) { + def ppIds = parameterSets.collect{it[0]} + assert ppIds.size() == ppIds.unique().size() : "All argument sets should have unique ids. Detected ids: $ppIds" +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_getChild.nf' + +// helper functions for reading params from file // +def _getChild(parent, child) { + if (child.contains("://") || java.nio.file.Paths.get(child).isAbsolute()) { + child + } else { + def parentAbsolute = java.nio.file.Paths.get(parent).toAbsolutePath().toString() + parentAbsolute.replaceAll('/[^/]*$', "/") + child + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_parseParamList.nf' +/** + * Figure out the param list format based on the file extension + * + * @param param_list A String containing the path to the parameter list file. + * + * @return A String containing the format of the parameter list file. + */ +def _paramListGuessFormat(param_list) { + if (param_list !instanceof String) { + "asis" + } else if (param_list.endsWith(".csv")) { + "csv" + } else if (param_list.endsWith(".json") || param_list.endsWith(".jsn")) { + "json" + } else if (param_list.endsWith(".yaml") || param_list.endsWith(".yml")) { + "yaml" + } else { + "yaml_blob" + } +} + + +/** + * Read the param list + * + * @param param_list One of the following: + * - A String containing the path to the parameter list file (csv, json or yaml), + * - A yaml blob of a list of maps (yaml_blob), + * - Or a groovy list of maps (asis). + * @param config A Map of the Viash configuration. + * + * @return A List of Maps containing the parameters. + */ +def _parseParamList(param_list, Map config) { + // first determine format by extension + def paramListFormat = _paramListGuessFormat(param_list) + + def paramListPath = (paramListFormat != "asis" && paramListFormat != "yaml_blob") ? + file(param_list, hidden: true) : + null + + // get the correct parser function for the detected params_list format + def paramSets = [] + if (paramListFormat == "asis") { + paramSets = param_list + } else if (paramListFormat == "yaml_blob") { + paramSets = readYamlBlob(param_list) + } else if (paramListFormat == "yaml") { + paramSets = readYaml(paramListPath) + } else if (paramListFormat == "json") { + paramSets = readJson(paramListPath) + } else if (paramListFormat == "csv") { + paramSets = readCsv(paramListPath) + } else { + error "Format of provided --param_list not recognised.\n" + + "Found: '$paramListFormat'.\n" + + "Expected: a csv file, a json file, a yaml file,\n" + + "a yaml blob or a groovy list of maps." + } + + // data checks + assert paramSets instanceof List: "--param_list should contain a list of maps" + for (value in paramSets) { + assert value instanceof Map: "--param_list should contain a list of maps" + } + + // id is argument + def idIsArgument = config.allArguments.any{it.plainName == "id"} + + // Reformat from List to List> by adding the ID as first element of a Tuple2 + paramSets = paramSets.collect({ data -> + def id = data.id + if (!idIsArgument) { + data = data.findAll{k, v -> k != "id"} + } + [id, data] + }) + + // Split parameters with 'multiple: true' + paramSets = paramSets.collect({ id, data -> + data = _splitParams(data, config) + [id, data] + }) + + // The paths of input files inside a param_list file may have been specified relatively to the + // location of the param_list file. These paths must be made absolute. + if (paramListPath) { + paramSets = paramSets.collect({ id, data -> + def new_data = data.collectEntries{ parName, parValue -> + def par = config.allArguments.find{it.plainName == parName} + if (par && par.type == "file" && par.direction == "input") { + if (parValue instanceof Collection) { + parValue = parValue.collectMany{path -> + def x = _resolveSiblingIfNotAbsolute(path, paramListPath) + x instanceof Collection ? x : [x] + } + } else { + parValue = _resolveSiblingIfNotAbsolute(parValue, paramListPath) + } + } + [parName, parValue] + } + [id, new_data] + }) + } + + return paramSets +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/_splitParams.nf' +/** + * Split parameters for arguments that accept multiple values using their separator + * + * @param paramList A Map containing parameters to split. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A Map of parameters where the parameter values have been split into a list using + * their seperator. + */ +Map _splitParams(Map parValues, Map config){ + def parsedParamValues = parValues.collectEntries { parName, parValue -> + def parameterSettings = config.allArguments.find({it.plainName == parName}) + + if (!parameterSettings) { + // if argument is not found, do not alter + return [parName, parValue] + } + if (parameterSettings.multiple) { // Check if parameter can accept multiple values + if (parValue instanceof Collection) { + parValue = parValue.collect{it instanceof String ? it.split(parameterSettings.multiple_sep) : it } + } else if (parValue instanceof String) { + parValue = parValue.split(parameterSettings.multiple_sep) + } else if (parValue == null) { + parValue = [] + } else { + parValue = [ parValue ] + } + parValue = parValue.flatten() + } + // For all parameters check if multiple values are only passed for + // arguments that allow it. Quietly simplify lists of length 1. + if (!parameterSettings.multiple && parValue instanceof Collection) { + assert parValue.size() == 1 : + "Error: argument ${parName} has too many values.\n" + + " Expected amount: 1. Found: ${parValue.size()}" + parValue = parValue[0] + } + [parName, parValue] + } + return parsedParamValues +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/channelFromParams.nf' +/** + * Parse nextflow parameters based on settings defined in a viash config. + * Return a list of parameter sets, each parameter set corresponding to + * an event in a nextflow channel. The output from this function can be used + * with Channel.fromList to create a nextflow channel with Vdsl3 formatted + * events. + * + * This function performs: + * - A filtering of the params which can be found in the config file. + * - Process the params_list argument which allows a user to to initialise + * a Vsdl3 channel with multiple parameter sets. Possible formats are + * csv, json, yaml, or simply a yaml_blob. A csv should have column names + * which correspond to the different arguments of this pipeline. A json or a yaml + * file should be a list of maps, each of which has keys corresponding to the + * arguments of the pipeline. A yaml blob can also be passed directly as a parameter. + * When passing a csv, json or yaml, relative path names are relativized to the + * location of the parameter file. + * - Combine the parameter sets into a vdsl3 Channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A list of parameters with the first element of the event being + * the event ID and the second element containing a map of the parsed parameters. + */ + +private List>> _paramsToParamSets(Map params, Map config){ + // todo: fetch key from run args + def key_ = config.name + + /* parse regular parameters (not in param_list) */ + /*************************************************/ + def globalParams = config.allArguments + .findAll { params.containsKey(it.plainName) } + .collectEntries { [ it.plainName, params[it.plainName] ] } + def globalID = params.get("id", null) + + /* process params_list arguments */ + /*********************************/ + def paramList = params.containsKey("param_list") && params.param_list != null ? + params.param_list : [] + // if (paramList instanceof String) { + // paramList = [paramList] + // } + // def paramSets = paramList.collectMany{ _parseParamList(it, config) } + // TODO: be able to process param_list when it is a list of strings + def paramSets = _parseParamList(paramList, config) + if (paramSets.isEmpty()) { + paramSets = [[null, [:]]] + } + + /* combine arguments into channel */ + /**********************************/ + def processedParams = paramSets.indexed().collect{ index, tup -> + // Process ID + def id = tup[0] ?: globalID + + if (workflow.stubRun && !id) { + // if stub run, explicitly add an id if missing + id = "stub${index}" + } + assert id != null: "Each parameter set should have at least an 'id'" + + // Process params + def parValues = globalParams + tup[1] + // // Remove parameters which are null, if the default is also null + // parValues = parValues.collectEntries{paramName, paramValue -> + // parameterSettings = config.functionality.allArguments.find({it.plainName == paramName}) + // if ( paramValue != null || parameterSettings.get("default", null) != null ) { + // [paramName, paramValue] + // } + // } + parValues = parValues.collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + assert par != null : "Error in module '${key_}' id '${id}': '${name}' is not a valid input argument" + + if (par == null) { + return [:] + } + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + + [ name, value ] + } + + [id, parValues] + } + + // Check if ids (first element of each list) is unique + _checkUniqueIds(processedParams) + return processedParams +} + +/** + * Parse nextflow parameters based on settings defined in a viash config + * and return a nextflow channel. + * + * @param params Input parameters. Can optionaly contain a 'param_list' key that + * provides a list of arguments that can be split up into multiple events + * in the output channel possible formats of param_lists are: a csv file, + * json file, a yaml file or a yaml blob. Each parameters set (event) must + * have a unique ID. + * @param config A Map of the Viash configuration. This Map can be generated from the config file + * using the readConfig() function. + * + * @return A nextflow Channel with events. Events are formatted as a tuple that contains + * first contains the ID of the event and as second element holds a parameter map. + * + * + */ +def channelFromParams(Map params, Map config) { + def processedParams = _paramsToParamSets(params, config) + return Channel.fromList(processedParams) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/checkUniqueIds.nf' +def checkUniqueIds(Map args) { + def stopOnError = args.stopOnError == null ? args.stopOnError : true + + def idChecker = new IDChecker() + + return filter { tup -> + if (!idChecker.observe(tup[0])) { + if (stopOnError) { + error "Duplicate id: ${tup[0]}" + } else { + log.warn "Duplicate id: ${tup[0]}, removing duplicate entry" + return false + } + } + return true + } +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/preprocessInputs.nf' +// This helper file will be deprecated soon +preprocessInputsDeprecationWarningPrinted = false + +def preprocessInputsDeprecationWarning() { + if (!preprocessInputsDeprecationWarningPrinted) { + preprocessInputsDeprecationWarningPrinted = true + System.err.println("Warning: preprocessInputs() is deprecated and will be removed in Viash 0.9.0.") + } +} + +/** + * Generate a nextflow Workflow that allows processing a channel of + * Vdsl3 formatted events and apply a Viash config to them: + * - Gather default parameters from the Viash config and make + * sure that they are correctly formatted (see applyConfig method). + * - Format the input parameters (also using the applyConfig method). + * - Apply the default parameter to the input parameters. + * - Do some assertions: + * ~ Check if the event IDs in the channel are unique. + * + * The events in the channel are formatted as tuples, with the + * first element of the tuples being a unique id of the parameter set, + * and the second element containg the the parameters themselves. + * Optional extra elements of the tuples will be passed to the output as is. + * + * @param args A map that must contain a 'config' key that points + * to a parsed config (see readConfig()). Optionally, a + * 'key' key can be provided which can be used to create a unique + * name for the workflow process. + * + * @return A workflow that allows processing a channel of Vdsl3 formatted events + * and apply a Viash config to them. + */ +def preprocessInputs(Map args) { + preprocessInputsDeprecationWarning() + + def config = args.config + assert config instanceof Map : + "Error in preprocessInputs: config must be a map. " + + "Expected class: Map. Found: config.getClass() is ${config.getClass()}" + def key_ = args.key ?: config.name + + // Get different parameter types (used throughout this function) + def defaultArgs = config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + map { tup -> + def id = tup[0] + def data = tup[1] + def passthrough = tup.drop(2) + + def new_data = (defaultArgs + data).collectEntries { name, value -> + def par = config.allArguments.find { it.plainName == name && (it.direction == "input" || it.type == "file") } + + if (par != null) { + value = _checkArgumentType("input", par, value, "in module '$key_' id '$id'") + } + + [ name, value ] + } + + [ id, new_data ] + passthrough + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runComponents.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component config. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component config. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component config. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component config. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runComponents(Map args) { + log.warn("runComponents is deprecated, use runEach instead") + assert args.components: "runComponents should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runComponents" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def id_ = args.id + + workflow runComponentsWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def comp_config = comp_.config + + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_config) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + // def new_id = id_(tup[0], tup[1], comp_config) + def new_id = tup[0] + if (id_ instanceof String) { + new_id = id_ + } else if (id_ instanceof Closure) { + new_id = id_(new_id, tup[1], comp_config) + } + [new_id] + tup.drop(1) + } + : filter_ch + def data_ch = id_ch | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_config) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_config) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + post_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runComponentsWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/runEach.nf' +/** + * Run a list of components on a stream of data. + * + * @param components: list of Viash VDSL3 modules to run + * @param fromState: a closure, a map or a list of keys to extract from the input data. + * If a closure, it will be called with the id, the data and the component itself. + * @param toState: a closure, a map or a list of keys to extract from the output data + * If a closure, it will be called with the id, the output data, the old state and the component itself. + * @param filter: filter function to apply to the input. + * It will be called with the id, the data and the component itself. + * @param id: id to use for the output data + * If a closure, it will be called with the id, the data and the component itself. + * @param auto: auto options to pass to the components + * + * @return: a workflow that runs the components + **/ +def runEach(Map args) { + assert args.components: "runEach should be passed a list of components to run" + + def components_ = args.components + if (components_ !instanceof List) { + components_ = [ components_ ] + } + assert components_.size() > 0: "pass at least one component to runEach" + + def fromState_ = args.fromState + def toState_ = args.toState + def filter_ = args.filter + def runIf_ = args.runIf + def id_ = args.id + + assert !runIf_ || runIf_ instanceof Closure: "runEach: must pass a Closure to runIf." + + workflow runEachWf { + take: input_ch + main: + + // generate one channel per method + out_chs = components_.collect{ comp_ -> + def filter_ch = filter_ + ? input_ch | filter{tup -> + filter_(tup[0], tup[1], comp_) + } + : input_ch + def id_ch = id_ + ? filter_ch | map{tup -> + def new_id = id_ + if (new_id instanceof Closure) { + new_id = new_id(tup[0], tup[1], comp_) + } + assert new_id instanceof String : "Error in runEach: id should be a String or a Closure that returns a String. Expected: id instanceof String. Found: ${new_id.getClass()}" + [new_id] + tup.drop(1) + } + : filter_ch + def chPassthrough = null + def chRun = null + if (runIf_) { + def idRunIfBranch = id_ch.branch{ tup -> + run: runIf_(tup[0], tup[1], comp_) + passthrough: true + } + chPassthrough = idRunIfBranch.passthrough + chRun = idRunIfBranch.run + } else { + chRun = id_ch + chPassthrough = Channel.empty() + } + def data_ch = chRun | map{tup -> + def new_data = tup[1] + if (fromState_ instanceof Map) { + new_data = fromState_.collectEntries{ key0, key1 -> + [key0, new_data[key1]] + } + } else if (fromState_ instanceof List) { + new_data = fromState_.collectEntries{ key -> + [key, new_data[key]] + } + } else if (fromState_ instanceof Closure) { + new_data = fromState_(tup[0], new_data, comp_) + } + tup.take(1) + [new_data] + tup.drop(1) + } + def out_ch = data_ch + | comp_.run( + auto: (args.auto ?: [:]) + [simplifyInput: false, simplifyOutput: false] + ) + def post_ch = toState_ + ? out_ch | map{tup -> + def output = tup[1] + def old_state = tup[2] + def new_state = null + if (toState_ instanceof Map) { + new_state = old_state + toState_.collectEntries{ key0, key1 -> + [key0, output[key1]] + } + } else if (toState_ instanceof List) { + new_state = old_state + toState_.collectEntries{ key -> + [key, output[key]] + } + } else if (toState_ instanceof Closure) { + new_state = toState_(tup[0], output, old_state, comp_) + } + [tup[0], new_state] + tup.drop(3) + } + : out_ch + + def return_ch = post_ch + | concat(chPassthrough) + + return_ch + } + + // mix all results + output_ch = + (out_chs.size == 1) + ? out_chs[0] + : out_chs[0].mix(*out_chs.drop(1)) + + emit: output_ch + } + + return runEachWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/channel/safeJoin.nf' +/** + * Join sourceChannel to targetChannel + * + * This function joins the sourceChannel to the targetChannel. + * However, each id in the targetChannel must be present in the + * sourceChannel. If _meta.join_id exists in the targetChannel, that is + * used as an id instead. If the id doesn't match any id in the sourceChannel, + * an error is thrown. + */ + +def safeJoin(targetChannel, sourceChannel, key) { + def sourceIDs = new IDChecker() + + def sourceCheck = sourceChannel + | map { tup -> + sourceIDs.observe(tup[0]) + tup + } + def targetCheck = targetChannel + | map { tup -> + def id = tup[0] + + if (!sourceIDs.contains(id)) { + error ( + "Error in module '${key}' when merging output with original state.\n" + + " Reason: output with id '${id}' could not be joined with source channel.\n" + + " If the IDs in the output channel differ from the input channel,\n" + + " please set `tup[1]._meta.join_id to the original ID.\n" + + " Original IDs in input channel: ['${sourceIDs.getItems().join("', '")}'].\n" + + " Unexpected ID in the output channel: '${id}'.\n" + + " Example input event: [\"id\", [input: file(...)]],\n" + + " Example output event: [\"newid\", [output: file(...), _meta: [join_id: \"id\"]]]" + ) + } + // TODO: add link to our documentation on how to fix this + + tup + } + + sourceCheck.cross(targetChannel) + | map{ left, right -> + right + left.drop(1) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/_processArgument.nf' +def _processArgument(arg) { + arg.multiple = arg.multiple != null ? arg.multiple : false + arg.required = arg.required != null ? arg.required : false + arg.direction = arg.direction != null ? arg.direction : "input" + arg.multiple_sep = arg.multiple_sep != null ? arg.multiple_sep : ";" + arg.plainName = arg.name.replaceAll("^-*", "") + + if (arg.type == "file") { + arg.must_exist = arg.must_exist != null ? arg.must_exist : true + arg.create_parent = arg.create_parent != null ? arg.create_parent : true + } + + // add default values to output files which haven't already got a default + if (arg.type == "file" && arg.direction == "output" && arg.default == null) { + def mult = arg.multiple ? "_*" : "" + def extSearch = "" + if (arg.default != null) { + extSearch = arg.default + } else if (arg.example != null) { + extSearch = arg.example + } + if (extSearch instanceof List) { + extSearch = extSearch[0] + } + def extSearchResult = extSearch.find("\\.[^\\.]+\$") + def ext = extSearchResult != null ? extSearchResult : "" + arg.default = "\$id.\$key.${arg.plainName}${mult}${ext}" + if (arg.multiple) { + arg.default = [arg.default] + } + } + + if (!arg.multiple) { + if (arg.default != null && arg.default instanceof List) { + arg.default = arg.default[0] + } + if (arg.example != null && arg.example instanceof List) { + arg.example = arg.example[0] + } + } + + if (arg.type == "boolean_true") { + arg.default = false + } + if (arg.type == "boolean_false") { + arg.default = true + } + + arg +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/addGlobalParams.nf' +def addGlobalArguments(config) { + def localConfig = [ + "argument_groups": [ + [ + "name": "Nextflow input-output arguments", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "arguments" : [ + [ + 'name': '--publish_dir', + 'required': true, + 'type': 'string', + 'description': 'Path to an output directory.', + 'example': 'output/', + 'multiple': false + ], + [ + 'name': '--param_list', + 'required': false, + 'type': 'string', + 'description': '''Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob. + | + |* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ ['id': 'foo', 'input': 'foo.txt'], ['id': 'bar', 'input': 'bar.txt'] ]`. + |* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`. + |* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]`. + |* A yaml blob can also be passed directly as a string. Example: `--param_list "[ {'id': 'foo', 'input': 'foo.txt'}, {'id': 'bar', 'input': 'bar.txt'} ]"`. + | + |When passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.'''.stripMargin(), + 'example': 'my_params.yaml', + 'multiple': false, + 'hidden': true + ] + // TODO: allow multiple: true in param_list? + // TODO: allow to specify a --param_list_regex to filter the param_list? + // TODO: allow to specify a --param_list_from_state to remap entries in the param_list? + ] + ] + ] + ] + + return processConfig(_mergeMap(config, localConfig)) +} + +def _mergeMap(Map lhs, Map rhs) { + return rhs.inject(lhs.clone()) { map, entry -> + if (map[entry.key] instanceof Map && entry.value instanceof Map) { + map[entry.key] = _mergeMap(map[entry.key], entry.value) + } else if (map[entry.key] instanceof Collection && entry.value instanceof Collection) { + map[entry.key] += entry.value + } else { + map[entry.key] = entry.value + } + return map + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/generateHelp.nf' +def _generateArgumentHelp(param) { + // alternatives are not supported + // def names = param.alternatives ::: List(param.name) + + def unnamedProps = [ + ["required parameter", param.required], + ["multiple values allowed", param.multiple], + ["output", param.direction.toLowerCase() == "output"], + ["file must exist", param.type == "file" && param.must_exist] + ].findAll{it[1]}.collect{it[0]} + + def dflt = null + if (param.default != null) { + if (param.default instanceof List) { + dflt = param.default.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + dflt = param.default.toString() + } + } + def example = null + if (param.example != null) { + if (param.example instanceof List) { + example = param.example.join(param.multiple_sep != null ? param.multiple_sep : ", ") + } else { + example = param.example.toString() + } + } + def min = param.min?.toString() + def max = param.max?.toString() + + def escapeChoice = { choice -> + def s1 = choice.replaceAll("\\n", "\\\\n") + def s2 = s1.replaceAll("\"", """\\\"""") + s2.contains(",") || s2 != choice ? "\"" + s2 + "\"" : s2 + } + def choices = param.choices == null ? + null : + "[ " + param.choices.collect{escapeChoice(it.toString())}.join(", ") + " ]" + + def namedPropsStr = [ + ["type", ([param.type] + unnamedProps).join(", ")], + ["default", dflt], + ["example", example], + ["choices", choices], + ["min", min], + ["max", max] + ] + .findAll{it[1]} + .collect{"\n " + it[0] + ": " + it[1].replaceAll("\n", "\\n")} + .join("") + + def descStr = param.description == null ? + "" : + _paragraphWrap("\n" + param.description.trim(), 80 - 8).join("\n ") + + "\n --" + param.plainName + + namedPropsStr + + descStr +} + +// Based on Helper.generateHelp() in Helper.scala +def _generateHelp(config) { + def fun = config + + // PART 1: NAME AND VERSION + def nameStr = fun.name + + (fun.version == null ? "" : " " + fun.version) + + // PART 2: DESCRIPTION + def descrStr = fun.description == null ? + "" : + "\n\n" + _paragraphWrap(fun.description.trim(), 80).join("\n") + + // PART 3: Usage + def usageStr = fun.usage == null ? + "" : + "\n\nUsage:\n" + fun.usage.trim() + + // PART 4: Options + def argGroupStrs = fun.allArgumentGroups.collect{argGroup -> + def name = argGroup.name + def descriptionStr = argGroup.description == null ? + "" : + "\n " + _paragraphWrap(argGroup.description.trim(), 80-4).join("\n ") + "\n" + def arguments = argGroup.arguments.collect{arg -> + arg instanceof String ? fun.allArguments.find{it.plainName == arg} : arg + }.findAll{it != null} + def argumentStrs = arguments.collect{param -> _generateArgumentHelp(param)} + + "\n\n$name:" + + descriptionStr + + argumentStrs.join("\n") + } + + // FINAL: combine + def out = nameStr + + descrStr + + usageStr + + argGroupStrs.join("") + + return out +} + +// based on Format._paragraphWrap +def _paragraphWrap(str, maxLength) { + def outLines = [] + str.split("\n").each{par -> + def words = par.split("\\s").toList() + + def word = null + def line = words.pop() + while(!words.isEmpty()) { + word = words.pop() + if (line.length() + word.length() + 1 <= maxLength) { + line = line + " " + word + } else { + outLines.add(line) + line = word + } + } + if (words.isEmpty()) { + outLines.add(line) + } + } + return outLines +} + +def helpMessage(config) { + if (params.containsKey("help") && params.help) { + def mergedConfig = addGlobalArguments(config) + def helpStr = _generateHelp(mergedConfig) + println(helpStr) + exit 0 + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/processConfig.nf' +def processConfig(config) { + // set defaults for arguments + config.arguments = + (config.arguments ?: []).collect{_processArgument(it)} + + // set defaults for argument_group arguments + config.argument_groups = + (config.argument_groups ?: []).collect{grp -> + grp.arguments = (grp.arguments ?: []).collect{_processArgument(it)} + grp + } + + // create combined arguments list + config.allArguments = + config.arguments + + config.argument_groups.collectMany{it.arguments} + + // add missing argument groups (based on Functionality::allArgumentGroups()) + def argGroups = config.argument_groups + if (argGroups.any{it.name.toLowerCase() == "arguments"}) { + argGroups = argGroups.collect{ grp -> + if (grp.name.toLowerCase() == "arguments") { + grp = grp + [ + arguments: grp.arguments + config.arguments + ] + } + grp + } + } else { + argGroups = argGroups + [ + name: "Arguments", + arguments: config.arguments + ] + } + config.allArgumentGroups = argGroups + + config +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/config/readConfig.nf' + +def readConfig(file) { + def config = readYaml(file ?: moduleDir.resolve("config.vsh.yaml")) + processConfig(config) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_resolveSiblingIfNotAbsolute.nf' +/** + * Resolve a path relative to the current file. + * + * @param str The path to resolve, as a String. + * @param parentPath The path to resolve relative to, as a Path. + * + * @return The path that may have been resovled, as a Path. + */ +def _resolveSiblingIfNotAbsolute(str, parentPath) { + if (str !instanceof String) { + return str + } + if (!_stringIsAbsolutePath(str)) { + return parentPath.resolveSibling(str) + } else { + return file(str, hidden: true) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/_stringIsAbsolutePath.nf' +/** + * Check whether a path as a string is absolute. + * + * In the past, we tried using `file(., relative: true).isAbsolute()`, + * but the 'relative' option was added in 22.10.0. + * + * @param path The path to check, as a String. + * + * @return Whether the path is absolute, as a boolean. + */ +def _stringIsAbsolutePath(path) { + def _resolve_URL_PROTOCOL = ~/^([a-zA-Z][a-zA-Z0-9]*:)?\\/.+/ + + assert path instanceof String + return _resolve_URL_PROTOCOL.matcher(path).matches() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/collectTraces.nf' +class CustomTraceObserver implements nextflow.trace.TraceObserver { + List traces + + CustomTraceObserver(List traces) { + this.traces = traces + } + + @Override + void onProcessComplete(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } + + @Override + void onProcessCached(nextflow.processor.TaskHandler handler, nextflow.trace.TraceRecord trace) { + def trace2 = trace.store.clone() + trace2.script = null + traces.add(trace2) + } +} + +def collectTraces() { + def traces = Collections.synchronizedList([]) + + // add custom trace observer which stores traces in the traces object + session.observers.add(new CustomTraceObserver(traces)) + + traces +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/deepClone.nf' +/** + * Performs a deep clone of the given object. + * @param x an object + */ +def deepClone(x) { + iterateMap(x, {it instanceof Cloneable ? it.clone() : it}) +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getPublishDir.nf' +def getPublishDir() { + return params.containsKey("publish_dir") ? params.publish_dir : + params.containsKey("publishDir") ? params.publishDir : + null +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/getRootDir.nf' + +// Recurse upwards until we find a '.build.yaml' file +def _findBuildYamlFile(pathPossiblySymlink) { + def path = pathPossiblySymlink.toRealPath() + def child = path.resolve(".build.yaml") + if (java.nio.file.Files.isDirectory(path) && java.nio.file.Files.exists(child)) { + return child + } else { + def parent = path.getParent() + if (parent == null) { + return null + } else { + return _findBuildYamlFile(parent) + } + } +} + +// get the root of the target folder +def getRootDir() { + def dir = _findBuildYamlFile(meta.resources_dir) + assert dir != null: "Could not find .build.yaml in the folder structure" + dir.getParent() +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/iterateMap.nf' +/** + * Recursively apply a function over the leaves of an object. + * @param obj The object to iterate over. + * @param fun The function to apply to each value. + * @return The object with the function applied to each value. + */ +def iterateMap(obj, fun) { + if (obj instanceof List && obj !instanceof String) { + return obj.collect{item -> + iterateMap(item, fun) + } + } else if (obj instanceof Map) { + return obj.collectEntries{key, item -> + [key.toString(), iterateMap(item, fun)] + } + } else { + return fun(obj) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/functions/niceView.nf' +/** + * A view for printing the event of each channel as a YAML blob. + * This is useful for debugging. + */ +def niceView() { + workflow niceViewWf { + take: input + main: + output = input + | view{toYamlBlob(it)} + emit: output + } + return niceViewWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readCsv.nf' + +def readCsv(file_path) { + def output = [] + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + + // todo: allow escaped quotes in string + // todo: allow single quotes? + def splitRegex = java.util.regex.Pattern.compile(''',(?=(?:[^"]*"[^"]*")*[^"]*$)''') + def removeQuote = java.util.regex.Pattern.compile('''"(.*)"''') + + def br = java.nio.file.Files.newBufferedReader(inputFile) + + def row = -1 + def header = null + while (br.ready() && header == null) { + def line = br.readLine() + row++ + if (!line.startsWith("#")) { + header = splitRegex.split(line, -1).collect{field -> + m = removeQuote.matcher(field) + m.find() ? m.replaceFirst('$1') : field + } + } + } + assert header != null: "CSV file should contain a header" + + while (br.ready()) { + def line = br.readLine() + row++ + if (line == null) { + br.close() + break + } + + if (!line.startsWith("#")) { + def predata = splitRegex.split(line, -1) + def data = predata.collect{field -> + if (field == "") { + return null + } + def m = removeQuote.matcher(field) + if (m.find()) { + return m.replaceFirst('$1') + } else { + return field + } + } + assert header.size() == data.size(): "Row $row should contain the same number as fields as the header" + + def dataMap = [header, data].transpose().collectEntries().findAll{it.value != null} + output.add(dataMap) + } + } + + output +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJson.nf' +def readJson(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parse(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readJsonBlob.nf' +def readJsonBlob(str) { + def jsonSlurper = new groovy.json.JsonSlurper() + jsonSlurper.parseText(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readTaggedYaml.nf' +// Custom constructor to modify how certain objects are parsed from YAML +class CustomConstructor extends org.yaml.snakeyaml.constructor.Constructor { + Path root + + class ConstructPath extends org.yaml.snakeyaml.constructor.AbstractConstruct { + public Object construct(org.yaml.snakeyaml.nodes.Node node) { + String filename = (String) constructScalar(node); + if (root != null) { + return root.resolve(filename); + } + return java.nio.file.Paths.get(filename); + } + } + + CustomConstructor(org.yaml.snakeyaml.LoaderOptions options, Path root) { + super(options) + this.root = root + // Handling !file tag and parse it back to a File type + this.yamlConstructors.put(new org.yaml.snakeyaml.nodes.Tag("!file"), new ConstructPath()) + } +} + +def readTaggedYaml(Path path) { + def options = new org.yaml.snakeyaml.LoaderOptions() + def constructor = new CustomConstructor(options, path.getParent()) + def yaml = new org.yaml.snakeyaml.Yaml(constructor) + return yaml.load(path.text) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYaml.nf' +def readYaml(file_path) { + def inputFile = file_path !instanceof Path ? file(file_path, hidden: true) : file_path + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(inputFile) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/readYamlBlob.nf' +def readYamlBlob(str) { + def yamlSlurper = new org.yaml.snakeyaml.Yaml() + yamlSlurper.load(str) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toJsonBlob.nf' +String toJsonBlob(data) { + return groovy.json.JsonOutput.toJson(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toTaggedYamlBlob.nf' +// Custom representer to modify how certain objects are represented in YAML +class CustomRepresenter extends org.yaml.snakeyaml.representer.Representer { + Path relativizer + + class RepresentPath implements org.yaml.snakeyaml.representer.Represent { + public String getFileName(Object obj) { + if (obj instanceof File) { + obj = ((File) obj).toPath(); + } + if (obj !instanceof Path) { + throw new IllegalArgumentException("Object: " + obj + " is not a Path or File"); + } + def path = (Path) obj; + + if (relativizer != null) { + return relativizer.relativize(path).toString() + } else { + return path.toString() + } + } + + public org.yaml.snakeyaml.nodes.Node representData(Object data) { + String filename = getFileName(data); + def tag = new org.yaml.snakeyaml.nodes.Tag("!file"); + return representScalar(tag, filename); + } + } + CustomRepresenter(org.yaml.snakeyaml.DumperOptions options, Path relativizer) { + super(options) + this.relativizer = relativizer + this.representers.put(sun.nio.fs.UnixPath, new RepresentPath()) + this.representers.put(Path, new RepresentPath()) + this.representers.put(File, new RepresentPath()) + } +} + +String toTaggedYamlBlob(data) { + return toRelativeTaggedYamlBlob(data, null) +} +String toRelativeTaggedYamlBlob(data, Path relativizer) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + def representer = new CustomRepresenter(options, relativizer) + def yaml = new org.yaml.snakeyaml.Yaml(representer, options) + return yaml.dump(data) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/toYamlBlob.nf' +String toYamlBlob(data) { + def options = new org.yaml.snakeyaml.DumperOptions() + options.setDefaultFlowStyle(org.yaml.snakeyaml.DumperOptions.FlowStyle.BLOCK) + options.setPrettyFlow(true) + def yaml = new org.yaml.snakeyaml.Yaml(options) + def cleanData = iterateMap(data, { it instanceof Path ? it.toString() : it }) + return yaml.dump(cleanData) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeJson.nf' +void writeJson(data, file) { + assert data: "writeJson: data should not be null" + assert file: "writeJson: file should not be null" + file.write(toJsonBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/readwrite/writeYaml.nf' +void writeYaml(data, file) { + assert data: "writeYaml: data should not be null" + assert file: "writeYaml: file should not be null" + file.write(toYamlBlob(data)) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/findStates.nf' +def findStates(Map params, Map config) { + def auto_config = deepClone(config) + def auto_params = deepClone(params) + + auto_config = auto_config.clone() + // override arguments + auto_config.argument_groups = [] + auto_config.arguments = [ + [ + type: "string", + name: "--id", + description: "A dummy identifier", + required: false + ], + [ + type: "file", + name: "--input_states", + example: "/path/to/input/directory/**/state.yaml", + description: "Path to input directory containing the datasets to be integrated.", + required: true, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--filter", + example: "foo/.*/state.yaml", + description: "Regex to filter state files by path.", + required: false + ], + // to do: make this a yaml blob? + [ + type: "string", + name: "--rename_keys", + example: ["newKey1:oldKey1", "newKey2:oldKey2"], + description: "Rename keys in the detected input files. This is useful if the input files do not match the set of input arguments of the workflow.", + required: false, + multiple: true, + multiple_sep: ";" + ], + [ + type: "string", + name: "--settings", + example: '{"output_dataset": "dataset.h5ad", "k": 10}', + description: "Global arguments as a JSON glob to be passed to all components.", + required: false + ] + ] + if (!(auto_params.containsKey("id"))) { + auto_params["id"] = "auto" + } + + // run auto config through processConfig once more + auto_config = processConfig(auto_config) + + workflow findStatesWf { + helpMessage(auto_config) + + output_ch = + channelFromParams(auto_params, auto_config) + | flatMap { autoId, args -> + + def globalSettings = args.settings ? readYamlBlob(args.settings) : [:] + + // look for state files in input dir + def stateFiles = args.input_states + + // filter state files by regex + if (args.filter) { + stateFiles = stateFiles.findAll{ stateFile -> + def stateFileStr = stateFile.toString() + def matcher = stateFileStr =~ args.filter + matcher.matches()} + } + + // read in states + def states = stateFiles.collect { stateFile -> + def state_ = readTaggedYaml(stateFile) + [state_.id, state_] + } + + // construct renameMap + if (args.rename_keys) { + def renameMap = args.rename_keys.collectEntries{renameString -> + def split = renameString.split(":") + assert split.size() == 2: "Argument 'rename_keys' should be of the form 'newKey:oldKey', or 'newKey:oldKey;newKey:oldKey' in case of multiple values" + split + } + + // rename keys in state, only let states through which have all keys + // also add global settings + states = states.collectMany{id, state -> + def newState = [:] + + for (key in renameMap.keySet()) { + def origKey = renameMap[key] + if (!(state.containsKey(origKey))) { + return [] + } + newState[key] = state[origKey] + } + + [[id, globalSettings + newState]] + } + } + + states + } + emit: + output_ch + } + + return findStatesWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/joinStates.nf' +def joinStates(Closure apply_) { + workflow joinStatesWf { + take: input_ch + main: + output_ch = input_ch + | toSortedList + | filter{ it.size() > 0 } + | map{ tups -> + def ids = tups.collect{it[0]} + def states = tups.collect{it[1]} + apply_(ids, states) + } + + emit: output_ch + } + return joinStatesWf +} +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishFiles.nf' +def publishFiles(Map args) { + def key_ = args.get("key") + + assert key_ != null : "publishFiles: key must be specified" + + workflow publishFilesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + def inputFiles_ = inputoutputFilenames_[0] + def outputFilenames_ = inputoutputFilenames_[1] + + [id_, inputFiles_, outputFilenames_] + } + | publishFilesProc + emit: input_ch + } + return publishFilesWf +} + +process publishFilesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), path(inputFiles, stageAs: "_inputfile?/*"), val(outputFiles) + output: + tuple val(id), path{outputFiles} + script: + def copyCommands = [ + inputFiles instanceof List ? inputFiles : [inputFiles], + outputFiles instanceof List ? outputFiles : [outputFiles] + ] + .transpose() + .collectMany{infile, outfile -> + if (infile.toString() != outfile.toString()) { + [ + "[ -d \"\$(dirname '${outfile.toString()}')\" ] || mkdir -p \"\$(dirname '${outfile.toString()}')\"", + "cp -r '${infile.toString()}' '${outfile.toString()}'" + ] + } else { + // no need to copy if infile is the same as outfile + [] + } + } + """ + echo "Copying output files to destination folder" + ${copyCommands.join("\n ")} + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishFilesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishFilesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishFilesByConfig: key must be specified" + + workflow publishFilesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + + // the processed state is a list of [key, value, inputPath, outputFilename] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - inputPath is a List[Path] + // - outputFilename is a List[String] + // - (inputPath, outputFilename) are the files that will be copied from src to dest (relative to the state.yaml) + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output OR multiple channels were emitted + // and the output was just not added to using the channel + // that is now being parsed + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[inputPath: [], outputFilename: []]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def inputPath = val instanceof File ? val.toPath() : val + [inputPath: inputPath, outputFilename: filename_ix] + } + def transposedOutputs = ["inputPath", "outputFilename"].collectEntries{ key -> + [key, outputPerFile.collect{dic -> dic[key]}] + } + return [[key: plainName_] + transposedOutputs] + } else { + def value_ = java.nio.file.Paths.get(filename) + def inputPath = value instanceof File ? value.toPath() : value + return [[inputPath: [inputPath], outputFilename: [filename]]] + } + } + + def inputPaths = processedState.collectMany{it.inputPath} + def outputFilenames = processedState.collectMany{it.outputFilename} + + + [id_, inputPaths, outputFilenames] + } + | publishFilesProc + emit: input_ch + } + return publishFilesSimpleWf +} + + + + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/publishStates.nf' +def collectFiles(obj) { + if (obj instanceof java.io.File || obj instanceof Path) { + return [obj] + } else if (obj instanceof List && obj !instanceof String) { + return obj.collectMany{item -> + collectFiles(item) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectFiles(item) + } + } else { + return [] + } +} + +/** + * Recurse through a state and collect all input files and their target output filenames. + * @param obj The state to recurse through. + * @param prefix The prefix to prepend to the output filenames. + */ +def collectInputOutputPaths(obj, prefix) { + if (obj instanceof File || obj instanceof Path) { + def path = obj instanceof Path ? obj : obj.toPath() + def ext = path.getFileName().toString().find("\\.[^\\.]+\$") ?: "" + def newFilename = prefix + ext + return [[obj, newFilename]] + } else if (obj instanceof List && obj !instanceof String) { + return obj.withIndex().collectMany{item, ix -> + collectInputOutputPaths(item, prefix + "_" + ix) + } + } else if (obj instanceof Map) { + return obj.collectMany{key, item -> + collectInputOutputPaths(item, prefix + "." + key) + } + } else { + return [] + } +} + +def publishStates(Map args) { + def key_ = args.get("key") + def yamlTemplate_ = args.get("output_state", args.get("outputState", '$id.$key.state.yaml')) + + assert key_ != null : "publishStates: key must be specified" + + workflow publishStatesWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] + + // the input files and the target output filenames + def inputoutputFilenames_ = collectInputOutputPaths(state_, id_ + "." + key_).transpose() + + def yamlFilename = yamlTemplate_ + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + + // TODO: do the pathnames in state_ match up with the outputFilenames_? + + // convert state to yaml blob + def yamlBlob_ = toRelativeTaggedYamlBlob([id: id_] + state_, java.nio.file.Paths.get(yamlFilename)) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesWf +} +process publishStatesProc { + // todo: check publishpath? + publishDir path: "${getPublishDir()}/", mode: "copy" + tag "$id" + input: + tuple val(id), val(yamlBlob), val(yamlFile) + output: + tuple val(id), path{[yamlFile]} + script: + """ + mkdir -p "\$(dirname '${yamlFile}')" + echo "Storing state as yaml" + cat > '${yamlFile}' << HERE +${yamlBlob} +HERE + """ +} + + +// this assumes that the state contains no other values other than those specified in the config +def publishStatesByConfig(Map args) { + def config = args.get("config") + assert config != null : "publishStatesByConfig: config must be specified" + + def key_ = args.get("key", config.name) + assert key_ != null : "publishStatesByConfig: key must be specified" + + workflow publishStatesSimpleWf { + take: input_ch + main: + input_ch + | map { tup -> + def id_ = tup[0] + def state_ = tup[1] // e.g. [output: new File("myoutput.h5ad"), k: 10] + def origState_ = tup[2] // e.g. [output: '$id.$key.foo.h5ad'] + + // TODO: allow overriding the state.yaml template + // TODO TODO: if auto.publish == "state", add output_state as an argument + def yamlTemplate = params.containsKey("output_state") ? params.output_state : '$id.$key.state.yaml' + def yamlFilename = yamlTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + def yamlDir = java.nio.file.Paths.get(yamlFilename).getParent() + + // the processed state is a list of [key, value] tuples, where + // - key is a String + // - value is any object that can be serialized to a Yaml (so a String/Integer/Long/Double/Boolean, a List, a Map, or a Path) + // - (key, value) are the tuples that will be saved to the state.yaml file + def processedState = + config.allArguments + .findAll { it.direction == "output" } + .collectMany { par -> + def plainName_ = par.plainName + // if the state does not contain the key, it's an + // optional argument for which the component did + // not generate any output + if (!state_.containsKey(plainName_)) { + return [] + } + def value = state_[plainName_] + // if the parameter is not a file, it should be stored + // in the state as-is, but is not something that needs + // to be copied from the source path to the dest path + if (par.type != "file") { + return [[key: plainName_, value: value]] + } + // if the orig state does not contain this filename, + // it's an optional argument for which the user specified + // that it should not be returned as a state + if (!origState_.containsKey(plainName_)) { + return [] + } + def filenameTemplate = origState_[plainName_] + // if the pararameter is multiple: true, fetch the template + if (par.multiple && filenameTemplate instanceof List) { + filenameTemplate = filenameTemplate[0] + } + // instantiate the template + def filename = filenameTemplate + .replaceAll('\\$id', id_) + .replaceAll('\\$\\{id\\}', id_) + .replaceAll('\\$key', key_) + .replaceAll('\\$\\{key\\}', key_) + if (par.multiple) { + // if the parameter is multiple: true, the filename + // should contain a wildcard '*' that is replaced with + // the index of the file + assert filename.contains("*") : "Module '${key_}' id '${id_}': Multiple output files specified, but no wildcard '*' in the filename: ${filename}" + def outputPerFile = value.withIndex().collect{ val, ix -> + def filename_ix = filename.replace("*", ix.toString()) + def value_ = java.nio.file.Paths.get(filename_ix) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + return value_ + } + return [["key": plainName_, "value": outputPerFile]] + } else { + def value_ = java.nio.file.Paths.get(filename) + // if id contains a slash + if (yamlDir != null) { + value_ = yamlDir.relativize(value_) + } + def inputPath = value instanceof File ? value.toPath() : value + return [["key": plainName_, value: value_]] + } + } + + + def updatedState_ = processedState.collectEntries{[it.key, it.value]} + + // convert state to yaml blob + def yamlBlob_ = toTaggedYamlBlob([id: id_] + updatedState_) + + [id_, yamlBlob_, yamlFilename] + } + | publishStatesProc + emit: input_ch + } + return publishStatesSimpleWf +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/states/setState.nf' +def setState(fun) { + assert fun instanceof Closure || fun instanceof Map || fun instanceof List : + "Error in setState: Expected process argument to be a Closure, a Map, or a List. Found: class ${fun.getClass()}" + + // if fun is a List, convert to map + if (fun instanceof List) { + // check whether fun is a list[string] + assert fun.every{it instanceof CharSequence} : "Error in setState: argument is a List, but not all elements are Strings" + fun = fun.collectEntries{[it, it]} + } + + // if fun is a map, convert to closure + if (fun instanceof Map) { + // check whether fun is a map[string, string] + assert fun.values().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all values are Strings" + assert fun.keySet().every{it instanceof CharSequence} : "Error in setState: argument is a Map, but not all keys are Strings" + def funMap = fun.clone() + // turn the map into a closure to be used later on + fun = { id_, state_ -> + assert state_ instanceof Map : "Error in setState: the state is not a Map" + funMap.collectMany{newkey, origkey -> + if (state_.containsKey(origkey)) { + [[newkey, state_[origkey]]] + } else { + [] + } + }.collectEntries() + } + } + + map { tup -> + def id = tup[0] + def state = tup[1] + def unfilteredState = fun(id, state) + def newState = unfilteredState.findAll{key, val -> val != null} + [id, newState] + tup.drop(2) + } +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processAuto.nf' +// TODO: unit test processAuto +def processAuto(Map auto) { + // remove null values + auto = auto.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = ["simplifyInput", "simplifyOutput", "transcript", "publish"] + def unexpectedKeys = auto.keySet() - expectedKeys + assert unexpectedKeys.isEmpty(), "unexpected keys in auto: '${unexpectedKeys.join("', '")}'" + + // check auto.simplifyInput + assert auto.simplifyInput instanceof Boolean, "auto.simplifyInput must be a boolean" + + // check auto.simplifyOutput + assert auto.simplifyOutput instanceof Boolean, "auto.simplifyOutput must be a boolean" + + // check auto.transcript + assert auto.transcript instanceof Boolean, "auto.transcript must be a boolean" + + // check auto.publish + assert auto.publish instanceof Boolean || auto.publish == "state", "auto.publish must be a boolean or 'state'" + + return auto.subMap(expectedKeys) +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processDirectives.nf' +def assertMapKeys(map, expectedKeys, requiredKeys, mapName) { + assert map instanceof Map : "Expected argument '$mapName' to be a Map. Found: class ${map.getClass()}" + map.forEach { key, val -> + assert key in expectedKeys : "Unexpected key '$key' in ${mapName ? mapName + " " : ""}map" + } + requiredKeys.forEach { requiredKey -> + assert map.containsKey(requiredKey) : "Missing required key '$key' in ${mapName ? mapName + " " : ""}map" + } +} + +// TODO: unit test processDirectives +def processDirectives(Map drctv) { + // remove null values + drctv = drctv.findAll{k, v -> v != null} + + // check for unexpected keys + def expectedKeys = [ + "accelerator", "afterScript", "beforeScript", "cache", "conda", "container", "containerOptions", "cpus", "disk", "echo", "errorStrategy", "executor", "machineType", "maxErrors", "maxForks", "maxRetries", "memory", "module", "penv", "pod", "publishDir", "queue", "label", "scratch", "storeDir", "stageInMode", "stageOutMode", "tag", "time" + ] + def unexpectedKeys = drctv.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Unexpected keys in process directive: '${unexpectedKeys.join("', '")}'" + + /* DIRECTIVE accelerator + accepted examples: + - [ limit: 4, type: "nvidia-tesla-k80" ] + */ + if (drctv.containsKey("accelerator")) { + assertMapKeys(drctv["accelerator"], ["type", "limit", "request", "runtime"], [], "accelerator") + } + + /* DIRECTIVE afterScript + accepted examples: + - "source /cluster/bin/cleanup" + */ + if (drctv.containsKey("afterScript")) { + assert drctv["afterScript"] instanceof CharSequence + } + + /* DIRECTIVE beforeScript + accepted examples: + - "source /cluster/bin/setup" + */ + if (drctv.containsKey("beforeScript")) { + assert drctv["beforeScript"] instanceof CharSequence + } + + /* DIRECTIVE cache + accepted examples: + - true + - false + - "deep" + - "lenient" + */ + if (drctv.containsKey("cache")) { + assert drctv["cache"] instanceof CharSequence || drctv["cache"] instanceof Boolean + if (drctv["cache"] instanceof CharSequence) { + assert drctv["cache"] in ["deep", "lenient"] : "Unexpected value for cache" + } + } + + /* DIRECTIVE conda + accepted examples: + - "bwa=0.7.15" + - "bwa=0.7.15 fastqc=0.11.5" + - ["bwa=0.7.15", "fastqc=0.11.5"] + */ + if (drctv.containsKey("conda")) { + if (drctv["conda"] instanceof List) { + drctv["conda"] = drctv["conda"].join(" ") + } + assert drctv["conda"] instanceof CharSequence + } + + /* DIRECTIVE container + accepted examples: + - "foo/bar:tag" + - [ registry: "reg", image: "im", tag: "ta" ] + is transformed to "reg/im:ta" + - [ image: "im" ] + is transformed to "im:latest" + */ + if (drctv.containsKey("container")) { + assert drctv["container"] instanceof Map || drctv["container"] instanceof CharSequence + if (drctv["container"] instanceof Map) { + def m = drctv["container"] + assertMapKeys(m, [ "registry", "image", "tag" ], ["image"], "container") + def part1 = + System.getenv('OVERRIDE_CONTAINER_REGISTRY') ? System.getenv('OVERRIDE_CONTAINER_REGISTRY') + "/" : + params.containsKey("override_container_registry") ? params["override_container_registry"] + "/" : // todo: remove? + m.registry ? m.registry + "/" : + "" + def part2 = m.image + def part3 = m.tag ? ":" + m.tag : ":latest" + drctv["container"] = part1 + part2 + part3 + } + } + + /* DIRECTIVE containerOptions + accepted examples: + - "--foo bar" + - ["--foo bar", "-f b"] + */ + if (drctv.containsKey("containerOptions")) { + if (drctv["containerOptions"] instanceof List) { + drctv["containerOptions"] = drctv["containerOptions"].join(" ") + } + assert drctv["containerOptions"] instanceof CharSequence + } + + /* DIRECTIVE cpus + accepted examples: + - 1 + - 10 + */ + if (drctv.containsKey("cpus")) { + assert drctv["cpus"] instanceof Integer + } + + /* DIRECTIVE disk + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("disk")) { + assert drctv["disk"] instanceof CharSequence + // assert drctv["disk"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE echo + accepted examples: + - true + - false + */ + if (drctv.containsKey("echo")) { + assert drctv["echo"] instanceof Boolean + } + + /* DIRECTIVE errorStrategy + accepted examples: + - "terminate" + - "finish" + */ + if (drctv.containsKey("errorStrategy")) { + assert drctv["errorStrategy"] instanceof CharSequence + assert drctv["errorStrategy"] in ["terminate", "finish", "ignore", "retry"] : "Unexpected value for errorStrategy" + } + + /* DIRECTIVE executor + accepted examples: + - "local" + - "sge" + */ + if (drctv.containsKey("executor")) { + assert drctv["executor"] instanceof CharSequence + assert drctv["executor"] in ["local", "sge", "uge", "lsf", "slurm", "pbs", "pbspro", "moab", "condor", "nqsii", "ignite", "k8s", "awsbatch", "google-pipelines"] : "Unexpected value for executor" + } + + /* DIRECTIVE machineType + accepted examples: + - "n1-highmem-8" + */ + if (drctv.containsKey("machineType")) { + assert drctv["machineType"] instanceof CharSequence + } + + /* DIRECTIVE maxErrors + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxErrors")) { + assert drctv["maxErrors"] instanceof Integer + } + + /* DIRECTIVE maxForks + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxForks")) { + assert drctv["maxForks"] instanceof Integer + } + + /* DIRECTIVE maxRetries + accepted examples: + - 1 + - 3 + */ + if (drctv.containsKey("maxRetries")) { + assert drctv["maxRetries"] instanceof Integer + } + + /* DIRECTIVE memory + accepted examples: + - "1 GB" + - "2TB" + - "3.2KB" + - "10.B" + */ + if (drctv.containsKey("memory")) { + assert drctv["memory"] instanceof CharSequence + // assert drctv["memory"].matches("[0-9]+(\\.[0-9]*)? *[KMGTPEZY]?B") + // ^ does not allow closures + } + + /* DIRECTIVE module + accepted examples: + - "ncbi-blast/2.2.27" + - "ncbi-blast/2.2.27:t_coffee/10.0" + - ["ncbi-blast/2.2.27", "t_coffee/10.0"] + */ + if (drctv.containsKey("module")) { + if (drctv["module"] instanceof List) { + drctv["module"] = drctv["module"].join(":") + } + assert drctv["module"] instanceof CharSequence + } + + /* DIRECTIVE penv + accepted examples: + - "smp" + */ + if (drctv.containsKey("penv")) { + assert drctv["penv"] instanceof CharSequence + } + + /* DIRECTIVE pod + accepted examples: + - [ label: "key", value: "val" ] + - [ annotation: "key", value: "val" ] + - [ env: "key", value: "val" ] + - [ [label: "l", value: "v"], [env: "e", value: "v"]] + */ + if (drctv.containsKey("pod")) { + if (drctv["pod"] instanceof Map) { + drctv["pod"] = [ drctv["pod"] ] + } + assert drctv["pod"] instanceof List + drctv["pod"].forEach { pod -> + assert pod instanceof Map + // TODO: should more checks be added? + // See https://www.nextflow.io/docs/latest/process.html?highlight=directives#pod + // e.g. does it contain 'label' and 'value', or 'annotation' and 'value', or ...? + } + } + + /* DIRECTIVE publishDir + accepted examples: + - [] + - [ [ path: "foo", enabled: true ], [ path: "bar", enabled: false ] ] + - "/path/to/dir" + is transformed to [[ path: "/path/to/dir" ]] + - [ path: "/path/to/dir", mode: "cache" ] + is transformed to [[ path: "/path/to/dir", mode: "cache" ]] + */ + // TODO: should we also look at params["publishDir"]? + if (drctv.containsKey("publishDir")) { + def pblsh = drctv["publishDir"] + + // check different options + assert pblsh instanceof List || pblsh instanceof Map || pblsh instanceof CharSequence + + // turn into list if not already so + // for some reason, 'if (!pblsh instanceof List) pblsh = [ pblsh ]' doesn't work. + pblsh = pblsh instanceof List ? pblsh : [ pblsh ] + + // check elements of publishDir + pblsh = pblsh.collect{ elem -> + // turn into map if not already so + elem = elem instanceof CharSequence ? [ path: elem ] : elem + + // check types and keys + assert elem instanceof Map : "Expected publish argument '$elem' to be a String or a Map. Found: class ${elem.getClass()}" + assertMapKeys(elem, [ "path", "mode", "overwrite", "pattern", "saveAs", "enabled" ], ["path"], "publishDir") + + // check elements in map + assert elem.containsKey("path") + assert elem["path"] instanceof CharSequence + if (elem.containsKey("mode")) { + assert elem["mode"] instanceof CharSequence + assert elem["mode"] in [ "symlink", "rellink", "link", "copy", "copyNoFollow", "move" ] + } + if (elem.containsKey("overwrite")) { + assert elem["overwrite"] instanceof Boolean + } + if (elem.containsKey("pattern")) { + assert elem["pattern"] instanceof CharSequence + } + if (elem.containsKey("saveAs")) { + assert elem["saveAs"] instanceof CharSequence //: "saveAs as a Closure is currently not supported. Surround your closure with single quotes to get the desired effect. Example: '\{ foo \}'" + } + if (elem.containsKey("enabled")) { + assert elem["enabled"] instanceof Boolean + } + + // return final result + elem + } + // store final directive + drctv["publishDir"] = pblsh + } + + /* DIRECTIVE queue + accepted examples: + - "long" + - "short,long" + - ["short", "long"] + */ + if (drctv.containsKey("queue")) { + if (drctv["queue"] instanceof List) { + drctv["queue"] = drctv["queue"].join(",") + } + assert drctv["queue"] instanceof CharSequence + } + + /* DIRECTIVE label + accepted examples: + - "big_mem" + - "big_cpu" + - ["big_mem", "big_cpu"] + */ + if (drctv.containsKey("label")) { + if (drctv["label"] instanceof CharSequence) { + drctv["label"] = [ drctv["label"] ] + } + assert drctv["label"] instanceof List + drctv["label"].forEach { label -> + assert label instanceof CharSequence + // assert label.matches("[a-zA-Z0-9]([a-zA-Z0-9_]*[a-zA-Z0-9])?") + // ^ does not allow closures + } + } + + /* DIRECTIVE scratch + accepted examples: + - true + - "/path/to/scratch" + - '$MY_PATH_TO_SCRATCH' + - "ram-disk" + */ + if (drctv.containsKey("scratch")) { + assert drctv["scratch"] == true || drctv["scratch"] instanceof CharSequence + } + + /* DIRECTIVE storeDir + accepted examples: + - "/path/to/storeDir" + */ + if (drctv.containsKey("storeDir")) { + assert drctv["storeDir"] instanceof CharSequence + } + + /* DIRECTIVE stageInMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageInMode")) { + assert drctv["stageInMode"] instanceof CharSequence + assert drctv["stageInMode"] in ["copy", "link", "symlink", "rellink"] + } + + /* DIRECTIVE stageOutMode + accepted examples: + - "copy" + - "link" + */ + if (drctv.containsKey("stageOutMode")) { + assert drctv["stageOutMode"] instanceof CharSequence + assert drctv["stageOutMode"] in ["copy", "move", "rsync"] + } + + /* DIRECTIVE tag + accepted examples: + - "foo" + - '$id' + */ + if (drctv.containsKey("tag")) { + assert drctv["tag"] instanceof CharSequence + } + + /* DIRECTIVE time + accepted examples: + - "1h" + - "2days" + - "1day 6hours 3minutes 30seconds" + */ + if (drctv.containsKey("time")) { + assert drctv["time"] instanceof CharSequence + // todo: validation regex? + } + + return drctv +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/processWorkflowArgs.nf' +def processWorkflowArgs(Map args, Map defaultWfArgs, Map meta) { + // override defaults with args + def workflowArgs = defaultWfArgs + args + + // check whether 'key' exists + assert workflowArgs.containsKey("key") : "Error in module '${meta.config.name}': key is a required argument" + + // if 'key' is a closure, apply it to the original key + if (workflowArgs["key"] instanceof Closure) { + workflowArgs["key"] = workflowArgs["key"](meta.config.name) + } + def key = workflowArgs["key"] + assert key instanceof CharSequence : "Expected process argument 'key' to be a String. Found: class ${key.getClass()}" + assert key ==~ /^[a-zA-Z_]\w*$/ : "Error in module '$key': Expected process argument 'key' to consist of only letters, digits or underscores. Found: ${key}" + + // check for any unexpected keys + def expectedKeys = ["key", "directives", "auto", "map", "mapId", "mapData", "mapPassthrough", "filter", "runIf", "fromState", "toState", "args", "renameKeys", "debug"] + def unexpectedKeys = workflowArgs.keySet() - expectedKeys + assert unexpectedKeys.isEmpty() : "Error in module '$key': unexpected arguments to the '.run()' function: '${unexpectedKeys.join("', '")}'" + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("directives") : "Error in module '$key': directives is a required argument" + assert workflowArgs["directives"] instanceof Map : "Error in module '$key': Expected process argument 'directives' to be a Map. Found: class ${workflowArgs['directives'].getClass()}" + workflowArgs["directives"] = processDirectives(defaultWfArgs.directives + workflowArgs["directives"]) + + // check whether directives exists and apply defaults + assert workflowArgs.containsKey("auto") : "Error in module '$key': auto is a required argument" + assert workflowArgs["auto"] instanceof Map : "Error in module '$key': Expected process argument 'auto' to be a Map. Found: class ${workflowArgs['auto'].getClass()}" + workflowArgs["auto"] = processAuto(defaultWfArgs.auto + workflowArgs["auto"]) + + // auto define publish, if so desired + if (workflowArgs.auto.publish == true && (workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : [:]).isEmpty()) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.publish is true, params.publish_dir needs to be defined.\n" + + // " Example: params.publish_dir = \"./output/\"" + def publishDir = getPublishDir() + + if (publishDir != null) { + workflowArgs.directives.publishDir = [[ + path: publishDir, + saveAs: "{ it.startsWith('.') ? null : it }", // don't publish hidden files, by default + mode: "copy" + ]] + } + } + + // auto define transcript, if so desired + if (workflowArgs.auto.transcript == true) { + // can't assert at this level thanks to the no_publish profile + // assert params.containsKey("transcriptsDir") || params.containsKey("transcripts_dir") || params.containsKey("publishDir") || params.containsKey("publish_dir") : + // "Error in module '${workflowArgs['key']}': if auto.transcript is true, either params.transcripts_dir or params.publish_dir needs to be defined.\n" + + // " Example: params.transcripts_dir = \"./transcripts/\"" + def transcriptsDir = + params.containsKey("transcripts_dir") ? params.transcripts_dir : + params.containsKey("transcriptsDir") ? params.transcriptsDir : + params.containsKey("publish_dir") ? params.publish_dir + "/_transcripts" : + params.containsKey("publishDir") ? params.publishDir + "/_transcripts" : + null + if (transcriptsDir != null) { + def timestamp = nextflow.Nextflow.getSession().getWorkflowMetadata().start.format('yyyy-MM-dd_HH-mm-ss') + def transcriptsPublishDir = [ + path: "$transcriptsDir/$timestamp/\${task.process.replaceAll(':', '-')}/\${id}/", + saveAs: "{ it.startsWith('.') ? it.replaceAll('^.', '') : null }", + mode: "copy" + ] + def publishDirs = workflowArgs.directives.publishDir != null ? workflowArgs.directives.publishDir : null ? workflowArgs.directives.publishDir : [] + workflowArgs.directives.publishDir = publishDirs + transcriptsPublishDir + } + } + + // if this is a stubrun, remove certain directives? + if (workflow.stubRun) { + workflowArgs.directives.keySet().removeAll(["publishDir", "cpus", "memory", "label"]) + } + + for (nam in ["map", "mapId", "mapData", "mapPassthrough", "filter", "runIf"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam]) { + assert workflowArgs[nam] instanceof Closure : "Error in module '$key': Expected process argument '$nam' to be null or a Closure. Found: class ${workflowArgs[nam].getClass()}" + } + } + + // TODO: should functions like 'map', 'mapId', 'mapData', 'mapPassthrough' be deprecated as well? + for (nam in ["map", "mapData", "mapPassthrough", "renameKeys"]) { + if (workflowArgs.containsKey(nam) && workflowArgs[nam] != null) { + log.warn "module '$key': workflow argument '$nam' is deprecated and will be removed in Viash 0.9.0. Please use 'fromState' and 'toState' instead." + } + } + + // check fromState + workflowArgs["fromState"] = _processFromState(workflowArgs.get("fromState"), key, meta.config) + + // check toState + workflowArgs["toState"] = _processToState(workflowArgs.get("toState"), key, meta.config) + + // return output + return workflowArgs +} + +def _processFromState(fromState, key_, config_) { + assert fromState == null || fromState instanceof Closure || fromState instanceof Map || fromState instanceof List : + "Error in module '$key_': Expected process argument 'fromState' to be null, a Closure, a Map, or a List. Found: class ${fromState.getClass()}" + if (fromState == null) { + return null + } + + // if fromState is a List, convert to map + if (fromState instanceof List) { + // check whether fromstate is a list[string] + assert fromState.every{it instanceof CharSequence} : "Error in module '$key_': fromState is a List, but not all elements are Strings" + fromState = fromState.collectEntries{[it, it]} + } + + // if fromState is a map, convert to closure + if (fromState instanceof Map) { + // check whether fromstate is a map[string, string] + assert fromState.values().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all values are Strings" + assert fromState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': fromState is a Map, but not all keys are Strings" + def fromStateMap = fromState.clone() + def requiredInputNames = meta.config.allArguments.findAll{it.required && it.direction == "Input"}.collect{it.plainName} + // turn the map into a closure to be used later on + fromState = { it -> + def state = it[1] + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def data = fromStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (state.containsKey(origkey)) { + [[newkey, state[origkey]]] + } else if (!requiredInputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': fromState key '$origkey' not found in current state") + } + }.collectEntries() + data + } + } + + return fromState +} + +def _processToState(toState, key_, config_) { + if (toState == null) { + toState = { tup -> tup[1] } + } + + // toState should be a closure, map[string, string], or list[string] + assert toState instanceof Closure || toState instanceof Map || toState instanceof List : + "Error in module '$key_': Expected process argument 'toState' to be a Closure, a Map, or a List. Found: class ${toState.getClass()}" + + // if toState is a List, convert to map + if (toState instanceof List) { + // check whether toState is a list[string] + assert toState.every{it instanceof CharSequence} : "Error in module '$key_': toState is a List, but not all elements are Strings" + toState = toState.collectEntries{[it, it]} + } + + // if toState is a map, convert to closure + if (toState instanceof Map) { + // check whether toState is a map[string, string] + assert toState.values().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all values are Strings" + assert toState.keySet().every{it instanceof CharSequence} : "Error in module '$key_': toState is a Map, but not all keys are Strings" + def toStateMap = toState.clone() + def requiredOutputNames = config_.allArguments.findAll{it.required && it.direction == "Output"}.collect{it.plainName} + // turn the map into a closure to be used later on + toState = { it -> + def output = it[1] + def state = it[2] + assert output instanceof Map : "Error in module '$key_': the output is not a Map" + assert state instanceof Map : "Error in module '$key_': the state is not a Map" + def extraEntries = toStateMap.collectMany{newkey, origkey -> + // check whether newkey corresponds to a required argument + if (output.containsKey(origkey)) { + [[newkey, output[origkey]]] + } else if (!requiredOutputNames.contains(origkey)) { + [] + } else { + throw new Exception("Error in module '$key_': toState key '$origkey' not found in current output") + } + }.collectEntries() + state + extraEntries + } + } + + return toState +} + +// helper file: 'src/main/resources/io/viash/runners/nextflow/workflowFactory/workflowFactory.nf' +def _debug(workflowArgs, debugKey) { + if (workflowArgs.debug) { + view { "process '${workflowArgs.key}' $debugKey tuple: $it" } + } else { + map { it } + } +} + +// depends on: innerWorkflowFactory +def workflowFactory(Map args, Map defaultWfArgs, Map meta) { + def workflowArgs = processWorkflowArgs(args, defaultWfArgs, meta) + def key_ = workflowArgs["key"] + def multipleArgs = meta.config.allArguments.findAll{ it.multiple }.collect{it.plainName} + + workflow workflowInstance { + take: input_ + + main: + def chModified = input_ + | checkUniqueIds([:]) + | _debug(workflowArgs, "input") + | map { tuple -> + tuple = deepClone(tuple) + + if (workflowArgs.map) { + tuple = workflowArgs.map(tuple) + } + if (workflowArgs.mapId) { + tuple[0] = workflowArgs.mapId(tuple[0]) + } + if (workflowArgs.mapData) { + tuple[1] = workflowArgs.mapData(tuple[1]) + } + if (workflowArgs.mapPassthrough) { + tuple = tuple.take(2) + workflowArgs.mapPassthrough(tuple.drop(2)) + } + + // check tuple + assert tuple instanceof List : + "Error in module '${key_}': element in channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + assert tuple.size() >= 2 : + "Error in module '${key_}': expected length of tuple in input channel to be two or greater.\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: tuple.size() == ${tuple.size()}" + + // check id field + if (tuple[0] instanceof GString) { + tuple[0] = tuple[0].toString() + } + assert tuple[0] instanceof CharSequence : + "Error in module '${key_}': first element of tuple in channel should be a String\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Found: ${tuple[0]}" + + // match file to input file + if (workflowArgs.auto.simplifyInput && (tuple[1] instanceof Path || tuple[1] instanceof List)) { + def inputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + + assert inputFiles.size() == 1 : + "Error in module '${key_}' id '${tuple[0]}'.\n" + + " Anonymous file inputs are only allowed when the process has exactly one file input.\n" + + " Expected: inputFiles.size() == 1. Found: inputFiles.size() is ${inputFiles.size()}" + + tuple[1] = [[ inputFiles[0].plainName, tuple[1] ]].collectEntries() + } + + // check data field + assert tuple[1] instanceof Map : + "Error in module '${key_}' id '${tuple[0]}': second element of tuple in channel should be a Map\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // rename keys of data field in tuple + if (workflowArgs.renameKeys) { + assert workflowArgs.renameKeys instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class: Map. Found: renameKeys.getClass() is ${workflowArgs.renameKeys.getClass()}" + assert tuple[1] instanceof Map : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Expected class: Map. Found: tuple[1].getClass() is ${tuple[1].getClass()}" + + // TODO: allow renameKeys to be a function? + workflowArgs.renameKeys.each { newKey, oldKey -> + assert newKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of newKey: String. Found: newKey.getClass() is ${newKey.getClass()}" + assert oldKey instanceof CharSequence : + "Error renaming data keys in module '${key_}' id '${tuple[0]}'.\n" + + " Example: renameKeys: ['new_key': 'old_key'].\n" + + " Expected class of oldKey: String. Found: oldKey.getClass() is ${oldKey.getClass()}" + assert tuple[1].containsKey(oldKey) : + "Error renaming data keys in module '${key}' id '${tuple[0]}'.\n" + + " Key '$oldKey' is missing in the data map. tuple[1].keySet() is '${tuple[1].keySet()}'" + tuple[1].put(newKey, tuple[1][oldKey]) + } + tuple[1].keySet().removeAll(workflowArgs.renameKeys.collect{ newKey, oldKey -> oldKey }) + } + tuple + } + + + def chRun = null + def chPassthrough = null + if (workflowArgs.runIf) { + def runIfBranch = chModified.branch{ tup -> + run: workflowArgs.runIf(tup[0], tup[1]) + passthrough: true + } + chRun = runIfBranch.run + chPassthrough = runIfBranch.passthrough + } else { + chRun = chModified + chPassthrough = Channel.empty() + } + + def chRunFiltered = workflowArgs.filter ? + chRun | filter{workflowArgs.filter(it)} : + chRun + + def chArgs = workflowArgs.fromState ? + chRunFiltered | map{ + def new_data = workflowArgs.fromState(it.take(2)) + [it[0], new_data] + } : + chRunFiltered | map {tup -> tup.take(2)} + + // fill in defaults + def chArgsWithDefaults = chArgs + | map { tuple -> + def id_ = tuple[0] + def data_ = tuple[1] + + // TODO: could move fromState to here + + // fetch default params from functionality + def defaultArgs = meta.config.allArguments + .findAll { it.containsKey("default") } + .collectEntries { [ it.plainName, it.default ] } + + // fetch overrides in params + def paramArgs = meta.config.allArguments + .findAll { par -> + def argKey = key_ + "__" + par.plainName + params.containsKey(argKey) + } + .collectEntries { [ it.plainName, params[key_ + "__" + it.plainName] ] } + + // fetch overrides in data + def dataArgs = meta.config.allArguments + .findAll { data_.containsKey(it.plainName) } + .collectEntries { [ it.plainName, data_[it.plainName] ] } + + // combine params + def combinedArgs = defaultArgs + paramArgs + workflowArgs.args + dataArgs + + // remove arguments with explicit null values + combinedArgs + .removeAll{_, val -> val == null || val == "viash_no_value" || val == "force_null"} + + combinedArgs = _processInputValues(combinedArgs, meta.config, id_, key_) + + [id_, combinedArgs] + tuple.drop(2) + } + + // TODO: move some of the _meta.join_id wrangling to the safeJoin() function. + def chInitialOutputMulti = chArgsWithDefaults + | _debug(workflowArgs, "processed") + // run workflow + | innerWorkflowFactory(workflowArgs) + def chInitialOutputList = chInitialOutputMulti instanceof List ? chInitialOutputMulti : [chInitialOutputMulti] + assert chInitialOutputList.size() > 0: "should have emitted at least one output channel" + // Add a channel ID to the events, which designates the channel the event was emitted from as a running number + // This number is used to sort the events later when the events are gathered from across the channels. + def chInitialOutputListWithIndexedEvents = chInitialOutputList.withIndex().collect{channel, channelIndex -> + def newChannel = channel + | map {tuple -> + assert tuple instanceof List : + "Error in module '${key_}': element in output channel should be a tuple [id, data, ...otherargs...]\n" + + " Example: [\"id\", [input: file('foo.txt'), arg: 10]].\n" + + " Expected class: List. Found: tuple.getClass() is ${tuple.getClass()}" + + def newEvent = [channelIndex] + tuple + return newEvent + } + return newChannel + } + // Put the events into 1 channel, cover case where there is only one channel is emitted + def chInitialOutput = chInitialOutputList.size() > 1 ? \ + chInitialOutputListWithIndexedEvents[0].mix(*chInitialOutputListWithIndexedEvents.tail()) : \ + chInitialOutputListWithIndexedEvents[0] + def chInitialOutputProcessed = chInitialOutput + | map { tuple -> + def channelId = tuple[0] + def id_ = tuple[1] + def output_ = tuple[2] + + // see if output map contains metadata + def meta_ = + output_ instanceof Map && output_.containsKey("_meta") ? + output_["_meta"] : + [:] + def join_id = meta_.join_id ?: id_ + + // remove metadata + output_ = output_.findAll{k, v -> k != "_meta"} + + // check value types + output_ = _checkValidOutputArgument(output_, meta.config, id_, key_) + + [join_id, channelId, id_, output_] + } + // | view{"chInitialOutput: ${it.take(3)}"} + + // join the output [prev_id, channel_id, new_id, output] with the previous state [prev_id, state, ...] + def chPublishWithPreviousState = safeJoin(chInitialOutputProcessed, chRunFiltered, key_) + // input tuple format: [join_id, channel_id, id, output, prev_state, ...] + // output tuple format: [join_id, channel_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(2).take(3)) + tup.take(3) + [new_state] + tup.drop(5) + } + if (workflowArgs.auto.publish == "state") { + def chPublishFiles = chPublishWithPreviousState + // input tuple format: [join_id, channel_id, id, new_state, ...] + // output tuple format: [join_id, channel_id, id, new_state] + | map{ tup -> + tup.take(4) + } + + safeJoin(chPublishFiles, chArgsWithDefaults, key_) + // input tuple format: [join_id, channel_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(2).take(3) + } + | publishFilesByConfig(key: key_, config: meta.config) + } + // Join the state from the events that were emitted from different channels + def chJoined = chInitialOutputProcessed + | map {tuple -> + def join_id = tuple[0] + def channel_id = tuple[1] + def id = tuple[2] + def other = tuple.drop(3) + // Below, groupTuple is used to join the events. To make sure resuming a workflow + // keeps working, the output state must be deterministic. This means the state needs to be + // sorted with groupTuple's has a 'sort' argument. This argument can be set to 'hash', + // but hashing the state when it is large can be problematic in terms of performance. + // Therefore, a custom comparator function is provided. We add the channel ID to the + // states so that we can use the channel ID to sort the items. + def stateWithChannelID = [[channel_id] * other.size(), other].transpose() + // A comparator that is provided to groupTuple's 'sort' argument is applied + // to all elements of the event tuple (that is not the 'id'). The comparator + // closure that is used below expects the input to be List. So the join_id and + // channel_id must also be wrapped in a list. + [[join_id], [channel_id], id] + stateWithChannelID + } + | groupTuple(by: 2, sort: {a, b -> a[0] <=> b[0]}, size: chInitialOutputList.size(), remainder: true) + | map {join_ids, _, id, statesWithChannelID -> + // Remove the channel IDs from the states + def states = statesWithChannelID.collect{it[1]} + def newJoinId = join_ids.flatten().unique{a, b -> a <=> b} + assert newJoinId.size() == 1: "Multiple events were emitted for '$id'." + def newJoinIdUnique = newJoinId[0] + + // Merge the states from the different channels + def newState = states.inject([:]){ old_state, state_to_add -> + return old_state + state_to_add.collectEntries{k, v -> + if (!multipleArgs.contains(k)) { + // if the key is not a multiple argument, we expect only one value + if (old_state.containsKey(k)) { + assert old_state[k] == v : "ID $id: multiple entries for argument $k were emitted." + } + [k, v] + } else { + // if the key is a multiple argument, append the different values into one list + def prevValue = old_state.getOrDefault(k, []) + def prevValueAsList = prevValue instanceof List ? prevValue : [prevValue] + [k, prevValueAsList + v] + } + } + } + + _checkAllRequiredOuputsPresent(newState, meta.config, id, key_) + + // simplify output if need be + if (workflowArgs.auto.simplifyOutput && newState.size() == 1) { + newState = newState.values()[0] + } + + return [newJoinIdUnique, id, newState] + } + + // join the output [prev_id, new_id, output] with the previous state [prev_id, state, ...] + def chNewState = safeJoin(chJoined, chRunFiltered, key_) + // input tuple format: [join_id, id, output, prev_state, ...] + // output tuple format: [join_id, id, new_state, ...] + | map{ tup -> + def new_state = workflowArgs.toState(tup.drop(1).take(3)) + tup.take(2) + [new_state] + tup.drop(4) + } + + if (workflowArgs.auto.publish == "state") { + def chPublishStates = chNewState + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [join_id, id, new_state] + | map{ tup -> + tup.take(3) + } + + safeJoin(chPublishStates, chArgsWithDefaults, key_) + // input tuple format: [join_id, id, new_state, orig_state, ...] + // output tuple format: [id, new_state, orig_state] + | map { tup -> + tup.drop(1).take(3) + } + | publishStatesByConfig(key: key_, config: meta.config) + } + chReturn = chNewState + | map { tup -> + // input tuple format: [join_id, id, new_state, ...] + // output tuple format: [id, new_state, ...] + tup.drop(1) + } + | _debug(workflowArgs, "output") + | concat(chPassthrough) + + emit: chReturn + } + + def wf = workflowInstance.cloneWithName(key_) + + // add factory function + wf.metaClass.run = { runArgs -> + workflowFactory(runArgs, workflowArgs, meta) + } + // add config to module for later introspection + wf.metaClass.config = meta.config + + return wf +} + +nextflow.enable.dsl=2 + +// START COMPONENT-SPECIFIC CODE + +// create meta object +meta = [ + "resources_dir": moduleDir.toRealPath().normalize(), + "config": processConfig(readJsonBlob('''{ + "name" : "join_graphs", + "namespace" : "neighbors", + "version" : "niche-compass", + "authors" : [ + { + "name" : "Jakub Majercik", + "roles" : [ + "maintainer" + ], + "info" : { + "role" : "Contributor", + "links" : { + "email" : "jakub@data-intuitive.com", + "github" : "jakubmajercik", + "linkedin" : "jakubmajercik" + }, + "organizations" : [ + { + "name" : "Data Intuitive", + "href" : "https://www.data-intuitive.com", + "role" : "Bioinformatics Engineer" + } + ] + } + } + ], + "argument_groups" : [ + { + "name" : "Inputs", + "arguments" : [ + { + "type" : "file", + "name" : "--input", + "description" : "Input H5MU file with pre-computed expression and spatial neighborhood graphs.", + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--modality", + "description" : "Modality from the input MuData file to process.", + "default" : [ + "rna" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--input_obsp_expression_graph", + "description" : "Key in `adata.obsp` containing the expression connectivity matrix.\n", + "default" : [ + "connectivities" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--input_obsp_spatial_graph", + "description" : "Key in `adata.obsp` containing the spatial connectivity matrix.\n", + "default" : [ + "spatial_connectivities" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Parameters", + "arguments" : [ + { + "type" : "double", + "name" : "--alpha", + "description" : "Weight of the spatial graph in the connection fusion. \nA value of 0 results in the original expression graph; \na value of 1 results in the spatial graph only.\n", + "default" : [ + 0.5 + ], + "required" : false, + "min" : 0.0, + "max" : 1.0, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + }, + { + "name" : "Outputs", + "arguments" : [ + { + "type" : "file", + "name" : "--output", + "description" : "Output H5MU file path.", + "example" : [ + "output.h5mu" + ], + "must_exist" : true, + "create_parent" : true, + "required" : true, + "direction" : "output", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--output_obsp_graph", + "description" : "Key under which to store the fused connectivity matrix in `adata.obsp`.\n", + "default" : [ + "spatial_expression_connectivities" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--output_compression", + "description" : "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", + "example" : [ + "gzip" + ], + "required" : false, + "choices" : [ + "gzip", + "lzf" + ], + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + } + ] + } + ], + "resources" : [ + { + "type" : "python_script", + "path" : "script.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/src/utils/setup_logger.py" + }, + { + "type" : "file", + "path" : "/src/workflows/utils/labels.config", + "dest" : "nextflow_labels.config" + } + ], + "description" : "Combine spatial and expression neighborhood graphs into a single graph for \nuse in downstream clustering or trajectory analysis.\n\nThe fusion is a linear combination of the expression connectivities \nand spatial connectivities matrices, weighted by `alpha`.\n\n$C_{joint} = (1 - \\\\alpha) \\\\cdot C_{expression} + \\\\alpha \\\\cdot C_{spatial}$\n", + "test_resources" : [ + { + "type" : "python_script", + "path" : "test.py", + "is_executable" : true + }, + { + "type" : "file", + "path" : "/resources_test/xenium/xenium_tiny.qc.all_neighbors.pca.h5mu" + } + ], + "status" : "enabled", + "scope" : { + "image" : "public", + "target" : "public" + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + }, + "runners" : [ + { + "type" : "executable", + "id" : "executable", + "docker_setup_strategy" : "ifneedbepullelsecachedbuild" + }, + { + "type" : "nextflow", + "id" : "nextflow", + "directives" : { + "label" : [ + "midcpu", + "midmem", + "middisk" + ], + "tag" : "$id" + }, + "auto" : { + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false + }, + "config" : { + "labels" : { + "mem1gb" : "memory = 1000000000.B", + "mem2gb" : "memory = 2000000000.B", + "mem5gb" : "memory = 5000000000.B", + "mem10gb" : "memory = 10000000000.B", + "mem20gb" : "memory = 20000000000.B", + "mem50gb" : "memory = 50000000000.B", + "mem100gb" : "memory = 100000000000.B", + "mem200gb" : "memory = 200000000000.B", + "mem500gb" : "memory = 500000000000.B", + "mem1tb" : "memory = 1000000000000.B", + "mem2tb" : "memory = 2000000000000.B", + "mem5tb" : "memory = 5000000000000.B", + "mem10tb" : "memory = 10000000000000.B", + "mem20tb" : "memory = 20000000000000.B", + "mem50tb" : "memory = 50000000000000.B", + "mem100tb" : "memory = 100000000000000.B", + "mem200tb" : "memory = 200000000000000.B", + "mem500tb" : "memory = 500000000000000.B", + "mem1gib" : "memory = 1073741824.B", + "mem2gib" : "memory = 2147483648.B", + "mem4gib" : "memory = 4294967296.B", + "mem8gib" : "memory = 8589934592.B", + "mem16gib" : "memory = 17179869184.B", + "mem32gib" : "memory = 34359738368.B", + "mem64gib" : "memory = 68719476736.B", + "mem128gib" : "memory = 137438953472.B", + "mem256gib" : "memory = 274877906944.B", + "mem512gib" : "memory = 549755813888.B", + "mem1tib" : "memory = 1099511627776.B", + "mem2tib" : "memory = 2199023255552.B", + "mem4tib" : "memory = 4398046511104.B", + "mem8tib" : "memory = 8796093022208.B", + "mem16tib" : "memory = 17592186044416.B", + "mem32tib" : "memory = 35184372088832.B", + "mem64tib" : "memory = 70368744177664.B", + "mem128tib" : "memory = 140737488355328.B", + "mem256tib" : "memory = 281474976710656.B", + "mem512tib" : "memory = 562949953421312.B", + "cpu1" : "cpus = 1", + "cpu2" : "cpus = 2", + "cpu5" : "cpus = 5", + "cpu10" : "cpus = 10", + "cpu20" : "cpus = 20", + "cpu50" : "cpus = 50", + "cpu100" : "cpus = 100", + "cpu200" : "cpus = 200", + "cpu500" : "cpus = 500", + "cpu1000" : "cpus = 1000" + }, + "script" : [ + "includeConfig(\\"nextflow_labels.config\\")" + ] + }, + "debug" : false, + "container" : "docker" + } + ], + "engines" : [ + { + "type" : "docker", + "id" : "docker", + "image" : "python:3.12-slim", + "target_registry" : "images.viash-hub.com", + "target_tag" : "niche-compass", + "namespace_separator" : "/", + "setup" : [ + { + "type" : "apt", + "packages" : [ + "procps" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "scanpy~=1.10.4", + "anndata~=0.12.7", + "awkward", + "mudata~=0.3.2" + ], + "script" : [ + "exec(\\"try:\\\\n import zarr; from importlib.metadata import version\\\\nexcept ModuleNotFoundError:\\\\n exit(0)\\\\nelse: assert int(version(\\\\\\"zarr\\\\\\").partition(\\\\\\".\\\\\\")[0]) > 2\\")" + ], + "upgrade" : true + } + ], + "test_setup" : [ + { + "type" : "apt", + "packages" : [ + "git" + ], + "interactive" : false + }, + { + "type" : "python", + "user" : false, + "packages" : [ + "viashpy==0.9.0" + ], + "github" : [ + "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils" + ], + "upgrade" : true + } + ] + }, + { + "type" : "native", + "id" : "native" + } + ], + "build_info" : { + "config" : "/workdir/root/repo/src/neighbors/join_graphs/config.vsh.yaml", + "runner" : "nextflow", + "engine" : "docker|native", + "output" : "/workdir/root/repo/target/nextflow/neighbors/join_graphs", + "viash_version" : "0.9.4", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", + "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" + }, + "package_config" : { + "name" : "openpipeline_spatial", + "version" : "niche-compass", + "info" : { + "test_resources" : [ + { + "type" : "s3", + "path" : "s3://openpipelines-bio/openpipeline_spatial/resources_test", + "dest" : "resources_test" + } + ] + }, + "repositories" : [ + { + "type" : "vsh", + "name" : "openpipeline", + "repo" : "openpipeline", + "tag" : "v4.0.3" + } + ], + "viash_version" : "0.9.4", + "source" : "/workdir/root/repo/src", + "target" : "/workdir/root/repo/target", + "config_mods" : [ + ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n.runners[.type == 'nextflow'].config.script := 'includeConfig(\\"nextflow_labels.config\\")'", + ".engines += { type: \\"native\\" }", + ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'", + ".engines[.type == 'docker'].target_tag := 'niche-compass'" + ], + "organization" : "vsh", + "links" : { + "repository" : "https://github.com/openpipelines-bio/openpipeline_spatial", + "docker_registry" : "ghcr.io" + } + } +}''')) +] + +// resolve dependencies dependencies (if any) + + +// inner workflow +// inner workflow hook +def innerWorkflowFactory(args) { + def rawScript = '''set -e +tempscript=".viash_script.py" +cat > "$tempscript" << VIASHMAIN +import sys +import mudata as mu + +## VIASH START +# The following code has been auto-generated by Viash. +par = { + 'input': $( if [ ! -z ${VIASH_PAR_INPUT+x} ]; then echo "r'${VIASH_PAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'input_obsp_expression_graph': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_EXPRESSION_GRAPH//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'input_obsp_spatial_graph': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_SPATIAL_GRAPH//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'alpha': $( if [ ! -z ${VIASH_PAR_ALPHA+x} ]; then echo "float(r'${VIASH_PAR_ALPHA//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'output': $( if [ ! -z ${VIASH_PAR_OUTPUT+x} ]; then echo "r'${VIASH_PAR_OUTPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'output_obsp_graph': $( if [ ! -z ${VIASH_PAR_OUTPUT_OBSP_GRAPH+x} ]; then echo "r'${VIASH_PAR_OUTPUT_OBSP_GRAPH//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'output_compression': $( if [ ! -z ${VIASH_PAR_OUTPUT_COMPRESSION+x} ]; then echo "r'${VIASH_PAR_OUTPUT_COMPRESSION//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ) +} +meta = { + 'name': $( if [ ! -z ${VIASH_META_NAME+x} ]; then echo "r'${VIASH_META_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'functionality_name': $( if [ ! -z ${VIASH_META_FUNCTIONALITY_NAME+x} ]; then echo "r'${VIASH_META_FUNCTIONALITY_NAME//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'resources_dir': $( if [ ! -z ${VIASH_META_RESOURCES_DIR+x} ]; then echo "r'${VIASH_META_RESOURCES_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'executable': $( if [ ! -z ${VIASH_META_EXECUTABLE+x} ]; then echo "r'${VIASH_META_EXECUTABLE//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'config': $( if [ ! -z ${VIASH_META_CONFIG+x} ]; then echo "r'${VIASH_META_CONFIG//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'temp_dir': $( if [ ! -z ${VIASH_META_TEMP_DIR+x} ]; then echo "r'${VIASH_META_TEMP_DIR//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'cpus': $( if [ ! -z ${VIASH_META_CPUS+x} ]; then echo "int(r'${VIASH_META_CPUS//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_b': $( if [ ! -z ${VIASH_META_MEMORY_B+x} ]; then echo "int(r'${VIASH_META_MEMORY_B//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kb': $( if [ ! -z ${VIASH_META_MEMORY_KB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mb': $( if [ ! -z ${VIASH_META_MEMORY_MB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gb': $( if [ ! -z ${VIASH_META_MEMORY_GB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tb': $( if [ ! -z ${VIASH_META_MEMORY_TB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pb': $( if [ ! -z ${VIASH_META_MEMORY_PB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_kib': $( if [ ! -z ${VIASH_META_MEMORY_KIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_KIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_mib': $( if [ ! -z ${VIASH_META_MEMORY_MIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_MIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_gib': $( if [ ! -z ${VIASH_META_MEMORY_GIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_GIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_tib': $( if [ ! -z ${VIASH_META_MEMORY_TIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_TIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), + 'memory_pib': $( if [ ! -z ${VIASH_META_MEMORY_PIB+x} ]; then echo "int(r'${VIASH_META_MEMORY_PIB//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ) +} +dep = { + +} + +## VIASH END + +sys.path.append(meta["resources_dir"]) +from setup_logger import setup_logger + +logger = setup_logger() + +## Read data +logger.info("Reading input data...") +adata = mu.read_h5ad(par["input"], mod=par["modality"]) + +## Validate inputs +spatial_key = par["input_obsp_spatial_graph"] +if spatial_key not in adata.obsp: + raise ValueError(f"Spatial graph key '{spatial_key}' not found in .obsp.") + +expr_key = par["input_obsp_expression_graph"] +if expr_key not in adata.obsp: + raise ValueError(f"Expression graph key '{expr_key}' not found in .obsp.") + +nn_graph_genes = adata.obsp[expr_key] +nn_graph_space = adata.obsp[spatial_key] + +## Combine graphs +alpha = par["alpha"] +logger.info( + f"Combining graphs (alpha={alpha}: spatial weight, " + f"{1 - alpha:.2f}: expression weight)..." +) +joint_graph = (1 - alpha) * nn_graph_genes + alpha * nn_graph_space +out_key = par["output_obsp_graph"] +logger.info(f"Storing result in .obsp['{out_key}']...") +adata.obsp[out_key] = joint_graph +adata.uns[out_key] = { + "params": {"alpha": alpha}, + "inputs": { + "expression_graph": expr_key, + "spatial_graph": spatial_key, + }, +} + +## Write output +logger.info("Saving output data...") +mdata = mu.MuData({par["modality"]: adata}) +mdata.write_h5mu(par["output"], compression=par["output_compression"]) +VIASHMAIN +python -B "$tempscript" +''' + + return vdsl3WorkflowFactory(args, meta, rawScript) +} + + + +/** + * Generate a workflow for VDSL3 modules. + * + * This function is called by the workflowFactory() function. + * + * Input channel: [id, input_map] + * Output channel: [id, output_map] + * + * Internally, this workflow will convert the input channel + * to a format which the Nextflow module will be able to handle. + */ +def vdsl3WorkflowFactory(Map args, Map meta, String rawScript) { + def key = args["key"] + def processObj = null + + workflow processWf { + take: input_ + main: + + if (processObj == null) { + processObj = _vdsl3ProcessFactory(args, meta, rawScript) + } + + output_ = input_ + | map { tuple -> + def id = tuple[0] + def data_ = tuple[1] + + if (workflow.stubRun) { + // add id if missing + data_ = [id: 'stub'] + data_ + } + + // process input files separately + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { par -> + def val = data_.containsKey(par.plainName) ? data_[par.plainName] : [] + def inputFiles = [] + if (val == null) { + inputFiles = [] + } else if (val instanceof List) { + inputFiles = val + } else if (val instanceof Path) { + inputFiles = [ val ] + } else { + inputFiles = [] + } + if (!workflow.stubRun) { + // throw error when an input file doesn't exist + inputFiles.each{ file -> + assert file.exists() : + "Error in module '${key}' id '${id}' argument '${par.plainName}'.\n" + + " Required input file does not exist.\n" + + " Path: '$file'.\n" + + " Expected input file to exist" + } + } + inputFiles + } + + // remove input files + def argsExclInputFiles = meta.config.allArguments + .findAll { (it.type != "file" || it.direction != "input") && data_.containsKey(it.plainName) } + .collectEntries { par -> + def parName = par.plainName + def val = data_[parName] + if (par.multiple && val instanceof Collection) { + val = val.join(par.multiple_sep) + } + if (par.direction == "output" && par.type == "file") { + val = val + .replaceAll('\\$id', id) + .replaceAll('\\$\\{id\\}', id) + .replaceAll('\\$key', key) + .replaceAll('\\$\\{key\\}', key) + } + [parName, val] + } + + [ id ] + inputPaths + [ argsExclInputFiles, meta.resources_dir ] + } + | processObj + | map { output -> + def outputFiles = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .indexed() + .collectEntries{ index, par -> + def out = output[index + 1] + // strip dummy '.exitcode' file from output (see nextflow-io/nextflow#2678) + if (!out instanceof List || out.size() <= 1) { + if (par.multiple) { + out = [] + } else { + assert !par.required : + "Error in module '${key}' id '${output[0]}' argument '${par.plainName}'.\n" + + " Required output file is missing" + out = null + } + } else if (out.size() == 2 && !par.multiple) { + out = out[1] + } else { + out = out.drop(1) + } + [ par.plainName, out ] + } + + // drop null outputs + outputFiles.removeAll{it.value == null} + + [ output[0], outputFiles ] + } + emit: output_ + } + + return processWf +} + +// depends on: session? +def _vdsl3ProcessFactory(Map workflowArgs, Map meta, String rawScript) { + // autodetect process key + def wfKey = workflowArgs["key"] + def procKeyPrefix = "${wfKey}_process" + def scriptMeta = nextflow.script.ScriptMeta.current() + def existing = scriptMeta.getProcessNames().findAll{it.startsWith(procKeyPrefix)} + def numbers = existing.collect{it.replace(procKeyPrefix, "0").toInteger()} + def newNumber = (numbers + [-1]).max() + 1 + + def procKey = newNumber == 0 ? procKeyPrefix : "$procKeyPrefix$newNumber" + + if (newNumber > 0) { + log.warn "Key for module '${wfKey}' is duplicated.\n", + "If you run a component multiple times in the same workflow,\n" + + "it's recommended you set a unique key for every call,\n" + + "for example: ${wfKey}.run(key: \"foo\")." + } + + // subset directives and convert to list of tuples + def drctv = workflowArgs.directives + + // TODO: unit test the two commands below + // convert publish array into tags + def valueToStr = { val -> + // ignore closures + if (val instanceof CharSequence) { + if (!val.matches('^[{].*[}]$')) { + '"' + val + '"' + } else { + val + } + } else if (val instanceof List) { + "[" + val.collect{valueToStr(it)}.join(", ") + "]" + } else if (val instanceof Map) { + "[" + val.collect{k, v -> k + ": " + valueToStr(v)}.join(", ") + "]" + } else { + val.inspect() + } + } + + // multiple entries allowed: label, publishdir + def drctvStrs = drctv.collect { key, value -> + if (key in ["label", "publishDir"]) { + value.collect{ val -> + if (val instanceof Map) { + "\n$key " + val.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else if (val == null) { + "" + } else { + "\n$key " + valueToStr(val) + } + }.join() + } else if (value instanceof Map) { + "\n$key " + value.collect{ k, v -> k + ": " + valueToStr(v) }.join(", ") + } else { + "\n$key " + valueToStr(value) + } + }.join() + + def inputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "input" } + .collect { ', path(viash_par_' + it.plainName + ', stageAs: "_viash_par/' + it.plainName + '_?/*")' } + .join() + + def outputPaths = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + // insert dummy into every output (see nextflow-io/nextflow#2678) + if (!par.multiple) { + ', path{[".exitcode", args.' + par.plainName + ']}' + } else { + ', path{[".exitcode"] + args.' + par.plainName + '}' + } + } + .join() + + // TODO: move this functionality somewhere else? + if (workflowArgs.auto.transcript) { + outputPaths = outputPaths + ', path{[".exitcode", ".command*"]}' + } else { + outputPaths = outputPaths + ', path{[".exitcode"]}' + } + + // create dirs for output files (based on BashWrapper.createParentFiles) + def createParentStr = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" && it.create_parent } + .collect { par -> + def contents = "args[\"${par.plainName}\"] instanceof List ? args[\"${par.plainName}\"].join('\" \"') : args[\"${par.plainName}\"]" + "\${ args.containsKey(\"${par.plainName}\") ? \"mkdir_parent '\" + escapeText(${contents}) + \"'\" : \"\" }" + } + .join("\n") + + // construct inputFileExports + def inputFileExports = meta.config.allArguments + .findAll { it.type == "file" && it.direction.toLowerCase() == "input" } + .collect { par -> + def contents = "viash_par_${par.plainName} instanceof List ? viash_par_${par.plainName}.join(\"${par.multiple_sep}\") : viash_par_${par.plainName}" + "\n\${viash_par_${par.plainName}.empty ? \"\" : \"export VIASH_PAR_${par.plainName.toUpperCase()}='\" + escapeText(${contents}) + \"'\"}" + } + + // NOTE: if using docker, use /tmp instead of tmpDir! + def tmpDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('VIASH_TMPDIR') ?: + System.getenv('VIASH_TEMPDIR') ?: + System.getenv('VIASH_TMP') ?: + System.getenv('TEMP') ?: + System.getenv('TMPDIR') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMP') ?: + '/tmp' + ).toAbsolutePath() + + // construct stub + def stub = meta.config.allArguments + .findAll { it.type == "file" && it.direction == "output" } + .collect { par -> + "\${ args.containsKey(\"${par.plainName}\") ? \"touch2 \\\"\" + (args[\"${par.plainName}\"] instanceof String ? args[\"${par.plainName}\"].replace(\"_*\", \"_0\") : args[\"${par.plainName}\"].join('\" \"')) + \"\\\"\" : \"\" }" + } + .join("\n") + + // escape script + def escapedScript = rawScript.replace('\\', '\\\\').replace('$', '\\$').replace('"""', '\\"\\"\\"') + + // publishdir assert + def assertStr = (workflowArgs.auto.publish == true) || workflowArgs.auto.transcript ? + """\nassert task.publishDir.size() > 0: "if auto.publish is true, params.publish_dir needs to be defined.\\n Example: --publish_dir './output/'" """ : + "" + + // generate process string + def procStr = + """nextflow.enable.dsl=2 + | + |def escapeText = { s -> s.toString().replaceAll("'", "'\\\"'\\\"'") } + |process $procKey {$drctvStrs + |input: + | tuple val(id)$inputPaths, val(args), path(resourcesDir, stageAs: ".viash_meta_resources") + |output: + | tuple val("\$id")$outputPaths, optional: true + |stub: + |\"\"\" + |touch2() { mkdir -p "\\\$(dirname "\\\$1")" && touch "\\\$1" ; } + |$stub + |\"\"\" + |script:$assertStr + |def parInject = args + | .findAll{key, value -> value != null} + | .collect{key, value -> "export VIASH_PAR_\${key.toUpperCase()}='\${escapeText(value)}'"} + | .join("\\n") + |\"\"\" + |# meta exports + |export VIASH_META_RESOURCES_DIR="\${resourcesDir}" + |export VIASH_META_TEMP_DIR="${['docker', 'podman', 'charliecloud'].any{ it == workflow.containerEngine } ? '/tmp' : tmpDir}" + |export VIASH_META_NAME="${meta.config.name}" + |# export VIASH_META_EXECUTABLE="\\\$VIASH_META_RESOURCES_DIR/\\\$VIASH_META_NAME" + |export VIASH_META_CONFIG="\\\$VIASH_META_RESOURCES_DIR/.config.vsh.yaml" + |\${task.cpus ? "export VIASH_META_CPUS=\$task.cpus" : "" } + |\${task.memory?.bytes != null ? "export VIASH_META_MEMORY_B=\$task.memory.bytes" : "" } + |if [ ! -z \\\${VIASH_META_MEMORY_B+x} ]; then + | export VIASH_META_MEMORY_KB=\\\$(( (\\\$VIASH_META_MEMORY_B+999) / 1000 )) + | export VIASH_META_MEMORY_MB=\\\$(( (\\\$VIASH_META_MEMORY_KB+999) / 1000 )) + | export VIASH_META_MEMORY_GB=\\\$(( (\\\$VIASH_META_MEMORY_MB+999) / 1000 )) + | export VIASH_META_MEMORY_TB=\\\$(( (\\\$VIASH_META_MEMORY_GB+999) / 1000 )) + | export VIASH_META_MEMORY_PB=\\\$(( (\\\$VIASH_META_MEMORY_TB+999) / 1000 )) + | export VIASH_META_MEMORY_KIB=\\\$(( (\\\$VIASH_META_MEMORY_B+1023) / 1024 )) + | export VIASH_META_MEMORY_MIB=\\\$(( (\\\$VIASH_META_MEMORY_KIB+1023) / 1024 )) + | export VIASH_META_MEMORY_GIB=\\\$(( (\\\$VIASH_META_MEMORY_MIB+1023) / 1024 )) + | export VIASH_META_MEMORY_TIB=\\\$(( (\\\$VIASH_META_MEMORY_GIB+1023) / 1024 )) + | export VIASH_META_MEMORY_PIB=\\\$(( (\\\$VIASH_META_MEMORY_TIB+1023) / 1024 )) + |fi + | + |# meta synonyms + |export VIASH_TEMP="\\\$VIASH_META_TEMP_DIR" + |export TEMP_DIR="\\\$VIASH_META_TEMP_DIR" + | + |# create output dirs if need be + |function mkdir_parent { + | for file in "\\\$@"; do + | mkdir -p "\\\$(dirname "\\\$file")" + | done + |} + |$createParentStr + | + |# argument exports${inputFileExports.join()} + |\$parInject + | + |# process script + |${escapedScript} + |\"\"\" + |} + |""".stripMargin() + + // TODO: print on debug + // if (workflowArgs.debug == true) { + // println("######################\n$procStr\n######################") + // } + + // write process to temp file + def tempFile = java.nio.file.Files.createTempFile("viash-process-${procKey}-", ".nf") + addShutdownHook { java.nio.file.Files.deleteIfExists(tempFile) } + tempFile.text = procStr + + // create process from temp file + def binding = new nextflow.script.ScriptBinding([:]) + def session = nextflow.Nextflow.getSession() + def parser = _getScriptLoader(session) + .setModule(true) + .setBinding(binding) + def moduleScript = parser.runScript(tempFile) + .getScript() + + // register module in meta + def module = new nextflow.script.IncludeDef.Module(name: procKey) + scriptMeta.addModule(moduleScript, module.name, module.alias) + + // retrieve and return process from meta + return scriptMeta.getProcess(procKey) +} + +// use Reflection to get a ScriptParser / ScriptLoader +// <25.02.0-edge: new nextflow.script.ScriptParser(session) +// >=25.02.0-edge: nextflow.script.ScriptLoaderFactory.create(session) +def _getScriptLoader(nextflow.Session session) { + // try using the old method + try { + Class scriptParserClass = Class.forName('nextflow.script.ScriptParser') + return scriptParserClass.getDeclaredConstructor(nextflow.Session).newInstance(session) + } catch (ClassNotFoundException e) { + // else try with the new method + try { + Class scriptLoaderFactoryClass = Class.forName('nextflow.script.ScriptLoaderFactory') + def createMethod = scriptLoaderFactoryClass.getDeclaredMethod('create', nextflow.Session) + return createMethod.invoke(null, session) // null because create is static + } catch (ClassNotFoundException | NoSuchMethodException | IllegalAccessException | java.lang.reflect.InvocationTargetException e2) { + // Handle the case where neither class is found + throw new Exception("Neither nextflow.script.ScriptParser nor nextflow.script.ScriptLoaderFactory could be found. Is this a compatible Nextflow version?", e2) + } + } +} + +// defaults +meta["defaults"] = [ + // key to be used to trace the process and determine output names + key: null, + + // fixed arguments to be passed to script + args: [:], + + // default directives + directives: readJsonBlob('''{ + "container" : { + "registry" : "images.viash-hub.com", + "image" : "vsh/openpipeline_spatial/neighbors/join_graphs", + "tag" : "niche-compass" + }, + "label" : [ + "midcpu", + "midmem", + "middisk" + ], + "tag" : "$id" +}'''), + + // auto settings + auto: readJsonBlob('''{ + "simplifyInput" : true, + "simplifyOutput" : false, + "transcript" : false, + "publish" : false +}'''), + + // Apply a map over the incoming tuple + // Example: `{ tup -> [ tup[0], [input: tup[1].output] ] + tup.drop(2) }` + map: null, + + // Apply a map over the ID element of a tuple (i.e. the first element) + // Example: `{ id -> id + "_foo" }` + mapId: null, + + // Apply a map over the data element of a tuple (i.e. the second element) + // Example: `{ data -> [ input: data.output ] }` + mapData: null, + + // Apply a map over the passthrough elements of a tuple (i.e. the tuple excl. the first two elements) + // Example: `{ pt -> pt.drop(1) }` + mapPassthrough: null, + + // Filter the channel + // Example: `{ tup -> tup[0] == "foo" }` + filter: null, + + // Choose whether or not to run the component on the tuple if the condition is true. + // Otherwise, the tuple will be passed through. + // Example: `{ tup -> tup[0] != "skip_this" }` + runIf: null, + + // Rename keys in the data field of the tuple (i.e. the second element) + // Will likely be deprecated in favour of `fromState`. + // Example: `[ "new_key": "old_key" ]` + renameKeys: null, + + // Fetch data from the state and pass it to the module without altering the current state. + // + // `fromState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be passed to the module as is. + // - If it is a `List[String]`, the data will be the values of the state at the given keys. + // - If it is a `Map[String, String]`, the data will be the values of the state at the given keys, with the keys renamed according to the map. + // - If it is a function, the tuple (`[id, state]`) in the channel will be passed to the function, and the result will be used as the data. + // + // Example: `{ id, state -> [input: state.fastq_file] }` + // Default: `null` + fromState: null, + + // Determine how the state should be updated after the module has been run. + // + // `toState` should be `null`, `List[String]`, `Map[String, String]` or a function. + // + // - If it is `null`, the state will be replaced with the output of the module. + // - If it is a `List[String]`, the state will be updated with the values of the data at the given keys. + // - If it is a `Map[String, String]`, the state will be updated with the values of the data at the given keys, with the keys renamed according to the map. + // - If it is a function, a tuple (`[id, output, state]`) will be passed to the function, and the result will be used as the new state. + // + // Example: `{ id, output, state -> state + [counts: state.output] }` + // Default: `{ id, output, state -> output }` + toState: null, + + // Whether or not to print debug messages + // Default: `false` + debug: false +] + +// initialise default workflow +meta["workflow"] = workflowFactory([key: meta.config.name], meta.defaults, meta) + +// add workflow to environment +nextflow.script.ScriptMeta.current().addDefinition(meta.workflow) + +// anonymous workflow for running this module as a standalone +workflow { + // add id argument if it's not already in the config + // TODO: deep copy + def newConfig = deepClone(meta.config) + def newParams = deepClone(params) + + def argsContainsId = newConfig.allArguments.any{it.plainName == "id"} + if (!argsContainsId) { + def idArg = [ + 'name': '--id', + 'required': false, + 'type': 'string', + 'description': 'A unique id for every entry.', + 'multiple': false + ] + newConfig.arguments.add(0, idArg) + newConfig = processConfig(newConfig) + } + if (!newParams.containsKey("id")) { + newParams.id = "run" + } + + helpMessage(newConfig) + + channelFromParams(newParams, newConfig) + // make sure id is not in the state if id is not in the args + | map {id, state -> + if (!argsContainsId) { + [id, state.findAll{k, v -> k != "id"}] + } else { + [id, state] + } + } + | meta.workflow.run( + auto: [ publish: "state" ] + ) +} + +// END COMPONENT-SPECIFIC CODE diff --git a/target/nextflow/neighbors/join_graphs/nextflow.config b/target/nextflow/neighbors/join_graphs/nextflow.config new file mode 100644 index 0000000..df2781a --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/nextflow.config @@ -0,0 +1,126 @@ +manifest { + name = 'neighbors/join_graphs' + mainScript = 'main.nf' + nextflowVersion = '!>=20.12.1-edge' + version = 'niche-compass' + description = 'Combine spatial and expression neighborhood graphs into a single graph for \nuse in downstream clustering or trajectory analysis.\n\nThe fusion is a linear combination of the expression connectivities \nand spatial connectivities matrices, weighted by `alpha`.\n\n$C_{joint} = (1 - \\alpha) \\cdot C_{expression} + \\alpha \\cdot C_{spatial}$\n' + author = 'Jakub Majercik' +} + +process.container = 'nextflow/bash:latest' + +// detect tempdir +tempDir = java.nio.file.Paths.get( + System.getenv('NXF_TEMP') ?: + System.getenv('VIASH_TEMP') ?: + System.getenv('TEMPDIR') ?: + System.getenv('TMPDIR') ?: + '/tmp' +).toAbsolutePath() + +profiles { + no_publish { + process { + withName: '.*' { + publishDir = [ + enabled: false + ] + } + } + } + mount_temp { + docker.temp = tempDir + podman.temp = tempDir + charliecloud.temp = tempDir + } + docker { + docker.enabled = true + // docker.userEmulation = true + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + singularity { + singularity.enabled = true + singularity.autoMounts = true + docker.enabled = false + podman.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + podman { + podman.enabled = true + docker.enabled = false + singularity.enabled = false + shifter.enabled = false + charliecloud.enabled = false + } + shifter { + shifter.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + charliecloud.enabled = false + } + charliecloud { + charliecloud.enabled = true + docker.enabled = false + singularity.enabled = false + podman.enabled = false + shifter.enabled = false + } +} + +process{ + withLabel: mem1gb { memory = 1000000000.B } + withLabel: mem2gb { memory = 2000000000.B } + withLabel: mem5gb { memory = 5000000000.B } + withLabel: mem10gb { memory = 10000000000.B } + withLabel: mem20gb { memory = 20000000000.B } + withLabel: mem50gb { memory = 50000000000.B } + withLabel: mem100gb { memory = 100000000000.B } + withLabel: mem200gb { memory = 200000000000.B } + withLabel: mem500gb { memory = 500000000000.B } + withLabel: mem1tb { memory = 1000000000000.B } + withLabel: mem2tb { memory = 2000000000000.B } + withLabel: mem5tb { memory = 5000000000000.B } + withLabel: mem10tb { memory = 10000000000000.B } + withLabel: mem20tb { memory = 20000000000000.B } + withLabel: mem50tb { memory = 50000000000000.B } + withLabel: mem100tb { memory = 100000000000000.B } + withLabel: mem200tb { memory = 200000000000000.B } + withLabel: mem500tb { memory = 500000000000000.B } + withLabel: mem1gib { memory = 1073741824.B } + withLabel: mem2gib { memory = 2147483648.B } + withLabel: mem4gib { memory = 4294967296.B } + withLabel: mem8gib { memory = 8589934592.B } + withLabel: mem16gib { memory = 17179869184.B } + withLabel: mem32gib { memory = 34359738368.B } + withLabel: mem64gib { memory = 68719476736.B } + withLabel: mem128gib { memory = 137438953472.B } + withLabel: mem256gib { memory = 274877906944.B } + withLabel: mem512gib { memory = 549755813888.B } + withLabel: mem1tib { memory = 1099511627776.B } + withLabel: mem2tib { memory = 2199023255552.B } + withLabel: mem4tib { memory = 4398046511104.B } + withLabel: mem8tib { memory = 8796093022208.B } + withLabel: mem16tib { memory = 17592186044416.B } + withLabel: mem32tib { memory = 35184372088832.B } + withLabel: mem64tib { memory = 70368744177664.B } + withLabel: mem128tib { memory = 140737488355328.B } + withLabel: mem256tib { memory = 281474976710656.B } + withLabel: mem512tib { memory = 562949953421312.B } + withLabel: cpu1 { cpus = 1 } + withLabel: cpu2 { cpus = 2 } + withLabel: cpu5 { cpus = 5 } + withLabel: cpu10 { cpus = 10 } + withLabel: cpu20 { cpus = 20 } + withLabel: cpu50 { cpus = 50 } + withLabel: cpu100 { cpus = 100 } + withLabel: cpu200 { cpus = 200 } + withLabel: cpu500 { cpus = 500 } + withLabel: cpu1000 { cpus = 1000 } +} + +includeConfig("nextflow_labels.config") diff --git a/target/nextflow/neighbors/join_graphs/nextflow_labels.config b/target/nextflow/neighbors/join_graphs/nextflow_labels.config new file mode 100644 index 0000000..541aaad --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/nextflow_labels.config @@ -0,0 +1,68 @@ +process { + // Default resources for components that hardly do any processing + memory = { 2.GB * task.attempt } + cpus = 1 + + // Retry for exit codes that have something to do with memory issues + errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' } + maxRetries = 3 + maxMemory = null + + // CPU resources + withLabel: singlecpu { cpus = 1 } + withLabel: lowcpu { cpus = 4 } + withLabel: midcpu { cpus = 10 } + withLabel: highcpu { cpus = 20 } + + // Memory resources + withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } } + withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } } + + // Disk space + // Nextflow apparently can't handle empty directives, i.e. + // withLabel: lowdisk {} + // so for that reason we have to add a dummy directive + withLabel: lowdisk { + dummyDirective = "dummyValue" + } + withLabel: middisk { + dummyDirective = "dummyValue" + } + withLabel: highdisk { + dummyDirective = "dummyValue" + } + withLabel: veryhighdisk { + dummyDirective = "dummyValue" + } + // NOTE: The above labels intentionally do not have an effect by default. + // The user should set the disk space requirements by adding the following + // to the compute environment: + // + // withLabel: lowdisk { disk = { 20.GB * task.attempt } } + // withLabel: middisk { disk = { 100.GB * task.attempt } } + // withLabel: highdisk { disk = { 200.GB * task.attempt } } + // withLabel: veryhighdisk { disk = { 500.GB * task.attempt } } +} + +def get_memory(to_compare) { + if (!process.containsKey("maxMemory") || !process.maxMemory) { + return to_compare + } + + try { + if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) { + return process.maxMemory + } + else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) { + return max_memory as nextflow.util.MemoryUnit + } + else { + return to_compare + } + } catch (all) { + println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!" + System.exit(1) + } +} diff --git a/target/nextflow/neighbors/join_graphs/nextflow_schema.json b/target/nextflow/neighbors/join_graphs/nextflow_schema.json new file mode 100644 index 0000000..0525769 --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/nextflow_schema.json @@ -0,0 +1,108 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "join_graphs", + "description": "Combine spatial and expression neighborhood graphs into a single graph for \nuse in downstream clustering or trajectory analysis.\n\nThe fusion is a linear combination of the expression connectivities \nand spatial connectivities matrices, weighted by `alpha`.\n\n$C_{joint} = (1 - \\alpha) \\cdot C_{expression} + \\alpha \\cdot C_{spatial}$\n", + "type": "object", + "$defs": { + "inputs": { + "title": "Inputs", + "type": "object", + "description": "No description", + "properties": { + "input": { + "type": "string", + "format": "path", + "exists": true, + "description": "Input H5MU file with pre-computed expression and spatial neighborhood graphs.", + "help_text": "Type: `file`, multiple: `False`, required, direction: `input`. " + }, + "modality": { + "type": "string", + "description": "Modality from the input MuData file to process.", + "help_text": "Type: `string`, multiple: `False`, default: `\"rna\"`. ", + "default": "rna" + }, + "input_obsp_expression_graph": { + "type": "string", + "description": "Key in `adata.obsp` containing the expression connectivity matrix.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"connectivities\"`. ", + "default": "connectivities" + }, + "input_obsp_spatial_graph": { + "type": "string", + "description": "Key in `adata.obsp` containing the spatial connectivity matrix.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"spatial_connectivities\"`. ", + "default": "spatial_connectivities" + } + } + }, + "outputs": { + "title": "Outputs", + "type": "object", + "description": "No description", + "properties": { + "output": { + "type": "string", + "format": "path", + "description": "Output H5MU file path.", + "help_text": "Type: `file`, multiple: `False`, required, default: `\"$id.$key.output.h5mu\"`, direction: `output`, example: `\"output.h5mu\"`. ", + "default": "$id.$key.output.h5mu" + }, + "output_obsp_graph": { + "type": "string", + "description": "Key under which to store the fused connectivity matrix in `adata.obsp`.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"spatial_expression_connectivities\"`. ", + "default": "spatial_expression_connectivities" + }, + "output_compression": { + "type": "string", + "description": "Compression format to use for the output AnnData and/or Mudata objects.\nBy default no compression is applied.\n", + "help_text": "Type: `string`, multiple: `False`, example: `\"gzip\"`, choices: ``gzip`, `lzf``. ", + "enum": [ + "gzip", + "lzf" + ] + } + } + }, + "parameters": { + "title": "Parameters", + "type": "object", + "description": "No description", + "properties": { + "alpha": { + "type": "number", + "description": "Weight of the spatial graph in the connection fusion", + "help_text": "Type: `double`, multiple: `False`, default: `0.5`. ", + "default": 0.5 + } + } + }, + "nextflow input-output arguments": { + "title": "Nextflow input-output arguments", + "type": "object", + "description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.", + "properties": { + "publish_dir": { + "type": "string", + "description": "Path to an output directory.", + "help_text": "Type: `string`, multiple: `False`, required, example: `\"output/\"`. " + } + } + } + }, + "allOf": [ + { + "$ref": "#/$defs/inputs" + }, + { + "$ref": "#/$defs/outputs" + }, + { + "$ref": "#/$defs/parameters" + }, + { + "$ref": "#/$defs/nextflow input-output arguments" + } + ] +} diff --git a/target/nextflow/neighbors/join_graphs/setup_logger.py b/target/nextflow/neighbors/join_graphs/setup_logger.py new file mode 100644 index 0000000..3ca1cdb --- /dev/null +++ b/target/nextflow/neighbors/join_graphs/setup_logger.py @@ -0,0 +1,12 @@ +def setup_logger(): + import logging + from sys import stdout + + logger = logging.getLogger() + logger.setLevel(logging.INFO) + console_handler = logging.StreamHandler(stdout) + logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s") + console_handler.setFormatter(logFormatter) + logger.addHandler(console_handler) + + return logger diff --git a/target/nextflow/neighbors/spatial_neighborhood_graph/.config.vsh.yaml b/target/nextflow/neighbors/spatial_neighborhood_graph/.config.vsh.yaml index 2e84801..85a4ac9 100644 --- a/target/nextflow/neighbors/spatial_neighborhood_graph/.config.vsh.yaml +++ b/target/nextflow/neighbors/spatial_neighborhood_graph/.config.vsh.yaml @@ -147,7 +147,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -270,7 +270,7 @@ build_info: output: "target/nextflow/neighbors/spatial_neighborhood_graph" executable: "target/nextflow/neighbors/spatial_neighborhood_graph/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -284,7 +284,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/neighbors/spatial_neighborhood_graph/main.nf b/target/nextflow/neighbors/spatial_neighborhood_graph/main.nf index be87a43..d08a78c 100644 --- a/target/nextflow/neighbors/spatial_neighborhood_graph/main.nf +++ b/target/nextflow/neighbors/spatial_neighborhood_graph/main.nf @@ -3220,7 +3220,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3373,7 +3373,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/neighbors/spatial_neighborhood_graph", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3393,7 +3393,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/nichecompass/nichecompass/.config.vsh.yaml b/target/nextflow/nichecompass/nichecompass/.config.vsh.yaml index 2ee4e89..ae07999 100644 --- a/target/nextflow/nichecompass/nichecompass/.config.vsh.yaml +++ b/target/nextflow/nichecompass/nichecompass/.config.vsh.yaml @@ -64,6 +64,17 @@ argument_groups: direction: "input" multiple: false multiple_sep: ";" + - type: "string" + name: "--var_input" + description: "Key in adata.var that indicates which features to include as input\ + \ for the NicheCompass model.\nIf not provided, all features will be included.\n" + info: null + default: + - "filter_with_hvg" + required: false + direction: "input" + multiple: false + multiple_sep: ";" - type: "string" name: "--input_obsp_spatial_connectivities" description: "Key in adata.obsp where connectivities of the spatial neighborhood\ @@ -851,6 +862,8 @@ resources: is_executable: true - type: "file" path: "setup_logger.py" +- type: "file" + path: "subset_vars.py" - type: "file" path: "nextflow_labels.config" dest: "nextflow_labels.config" @@ -875,7 +888,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -998,7 +1011,7 @@ build_info: output: "target/nextflow/nichecompass/nichecompass" executable: "target/nextflow/nichecompass/nichecompass/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" package_config: name: "openpipeline_spatial" @@ -1012,7 +1025,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/nichecompass/nichecompass/main.nf b/target/nextflow/nichecompass/nichecompass/main.nf index e20451a..58490e6 100644 --- a/target/nextflow/nichecompass/nichecompass/main.nf +++ b/target/nextflow/nichecompass/nichecompass/main.nf @@ -3109,6 +3109,18 @@ meta = [ "multiple" : false, "multiple_sep" : ";" }, + { + "type" : "string", + "name" : "--var_input", + "description" : "Key in adata.var that indicates which features to include as input for the NicheCompass model.\nIf not provided, all features will be included.\n", + "default" : [ + "filter_with_hvg" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" + }, { "type" : "string", "name" : "--input_obsp_spatial_connectivities", @@ -3943,6 +3955,10 @@ meta = [ "type" : "file", "path" : "/src/utils/setup_logger.py" }, + { + "type" : "file", + "path" : "/src/utils/subset_vars.py" + }, { "type" : "file", "path" : "/src/workflows/utils/labels.config", @@ -3979,7 +3995,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -4129,7 +4145,7 @@ meta = [ "engine" : "docker|native", "output" : "/workdir/root/repo/target/nextflow/nichecompass/nichecompass", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -4149,7 +4165,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", @@ -4194,6 +4210,7 @@ par = { 'input_gp_mask': $( if [ ! -z ${VIASH_PAR_INPUT_GP_MASK+x} ]; then echo "r'${VIASH_PAR_INPUT_GP_MASK//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), 'modality': $( if [ ! -z ${VIASH_PAR_MODALITY+x} ]; then echo "r'${VIASH_PAR_MODALITY//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), 'layer': $( if [ ! -z ${VIASH_PAR_LAYER+x} ]; then echo "r'${VIASH_PAR_LAYER//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), + 'var_input': $( if [ ! -z ${VIASH_PAR_VAR_INPUT+x} ]; then echo "r'${VIASH_PAR_VAR_INPUT//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), 'input_obsp_spatial_connectivities': $( if [ ! -z ${VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES+x} ]; then echo "r'${VIASH_PAR_INPUT_OBSP_SPATIAL_CONNECTIVITIES//\\'/\\'\\"\\'\\"r\\'}'"; else echo None; fi ), 'input_obs_covariates': $( if [ ! -z ${VIASH_PAR_INPUT_OBS_COVARIATES+x} ]; then echo "r'${VIASH_PAR_INPUT_OBS_COVARIATES//\\'/\\'\\"\\'\\"r\\'}'.split(';')"; else echo None; fi ), 'min_genes_per_gp': $( if [ ! -z ${VIASH_PAR_MIN_GENES_PER_GP+x} ]; then echo "int(r'${VIASH_PAR_MIN_GENES_PER_GP//\\'/\\'\\"\\'\\"r\\'}')"; else echo None; fi ), @@ -4288,6 +4305,7 @@ dep = { sys.path.append(meta["resources_dir"]) from setup_logger import setup_logger +from subset_vars import subset_vars logger = setup_logger() @@ -4300,6 +4318,11 @@ logger.info(f"GPU count: {torch.cuda.device_count()}") ## Read in data adata = mu.read_h5ad(par["input"], mod=par["modality"]) +# Subset to HVG +if par["var_input"]: + # Subset to HVG + adata = subset_vars(adata, subset_col=par["var_input"]).copy() + # Counts need to be float32 to be processed by nichecompass model # See https://discuss.pytorch.org/t/runtimeerror-mat1-and-mat2-must-have-the-same-dtype/166759 counts_dtype = ( diff --git a/target/nextflow/nichecompass/nichecompass/subset_vars.py b/target/nextflow/nichecompass/nichecompass/subset_vars.py new file mode 100644 index 0000000..4025000 --- /dev/null +++ b/target/nextflow/nichecompass/nichecompass/subset_vars.py @@ -0,0 +1,49 @@ +def subset_vars(adata, subset_col): + """ + Subset AnnData object on highly variable genes or a boolean mask. + + Parameters + ---------- + adata : AnnData + Annotated data object + subset_col : str, pd.Series, pd.Index, or np.ndarray + Name of the boolean column in `adata.var` that contains the information if features should be used or not, + or a boolean mask (same length as adata.var) + + Returns + ------- + AnnData + Copy of `adata` with subsetted features + """ + import pandas as pd + import numpy as np + + # Convert all input types to a pandas Series + if isinstance(subset_col, str): + if subset_col not in adata.var.columns: + raise ValueError( + f"Requested to use .var column '{subset_col}' as a selection of genes, but the column is not available." + ) + mask = adata.var[subset_col] + elif isinstance(subset_col, pd.Series): + mask = subset_col + elif isinstance(subset_col, (pd.Index, np.ndarray, list)): + mask = pd.Series(subset_col, index=adata.var.index) + else: + raise TypeError( + "subset_col must be a string (column name) or a boolean mask (Series, Index, ndarray, or list)." + ) + + # Validate mask + if not pd.api.types.is_bool_dtype(mask): + raise ValueError( + f"Expected mask to be boolean, but found {mask.dtype}. Can not subset data." + ) + if mask.isna().sum() > 0: + raise ValueError("Mask contains NaN values. Can not subset data.") + if len(mask) != adata.n_vars: + raise ValueError( + f"Mask length {len(mask)} does not match number of variables {adata.n_vars}." + ) + + return adata[:, mask].copy() diff --git a/target/nextflow/workflows/ingestion/spaceranger_mapping/.config.vsh.yaml b/target/nextflow/workflows/ingestion/spaceranger_mapping/.config.vsh.yaml index a6cd492..561182e 100644 --- a/target/nextflow/workflows/ingestion/spaceranger_mapping/.config.vsh.yaml +++ b/target/nextflow/workflows/ingestion/spaceranger_mapping/.config.vsh.yaml @@ -399,7 +399,7 @@ repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -477,7 +477,7 @@ build_info: output: "target/nextflow/workflows/ingestion/spaceranger_mapping" executable: "target/nextflow/workflows/ingestion/spaceranger_mapping/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - "target/nextflow/mapping/spaceranger_count" @@ -494,7 +494,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/workflows/ingestion/spaceranger_mapping/main.nf b/target/nextflow/workflows/ingestion/spaceranger_mapping/main.nf index 3bcca07..d44e94a 100644 --- a/target/nextflow/workflows/ingestion/spaceranger_mapping/main.nf +++ b/target/nextflow/workflows/ingestion/spaceranger_mapping/main.nf @@ -3532,7 +3532,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3623,7 +3623,7 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/ingestion/spaceranger_mapping", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3643,7 +3643,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", diff --git a/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml b/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml index 60558e1..d4220ed 100644 --- a/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml +++ b/target/nextflow/workflows/multiomics/spatial_process_samples/.config.vsh.yaml @@ -557,12 +557,12 @@ dependencies: repository: type: "vsh" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -640,10 +640,10 @@ build_info: output: "target/nextflow/workflows/multiomics/spatial_process_samples" executable: "target/nextflow/workflows/multiomics/spatial_process_samples/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - - "target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples" + - "target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples" package_config: name: "openpipeline_spatial" version: "niche-compass" @@ -656,7 +656,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf b/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf index e7ab9bb..0358d26 100644 --- a/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf +++ b/target/nextflow/workflows/multiomics/spatial_process_samples/main.nf @@ -3707,7 +3707,7 @@ meta = [ "repository" : { "type" : "vsh", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } } ], @@ -3716,7 +3716,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3807,7 +3807,7 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/multiomics/spatial_process_samples", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3827,7 +3827,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", @@ -3850,7 +3850,7 @@ meta = [ // resolve dependencies dependencies (if any) meta["root_dir"] = getRootDir() -include { process_samples as spatial_sample_processing_viashalias } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/process_samples/main.nf" +include { process_samples as spatial_sample_processing_viashalias } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/process_samples/main.nf" spatial_sample_processing = spatial_sample_processing_viashalias.run(key: "spatial_sample_processing") // inner workflow diff --git a/target/nextflow/workflows/niche/nichecompass_leiden/.config.vsh.yaml b/target/nextflow/workflows/niche/nichecompass_leiden/.config.vsh.yaml index bdcb09a..5a388b1 100644 --- a/target/nextflow/workflows/niche/nichecompass_leiden/.config.vsh.yaml +++ b/target/nextflow/workflows/niche/nichecompass_leiden/.config.vsh.yaml @@ -120,21 +120,21 @@ argument_groups: direction: "input" multiple: false multiple_sep: ";" -- name: "Sample ID options" - description: "Options for adding the id to .obs on the MuData object. Having a sample\ - \ \nid present in a requirement of several components for this pipeline.\n" - arguments: - - type: "boolean" - name: "--include_sample_as_covariate" - description: "Whether to include the sample information as a categorical covariate\ - \ for the \nNicheCompass model.\n" + - type: "string" + name: "--var_input" + description: "Key in adata.var that indicates which features to include as input\ + \ for the NicheCompass model.\nIf not provided, all features will be included.\n" info: null default: - - true + - "filter_with_hvg" required: false direction: "input" multiple: false multiple_sep: ";" +- name: "Sample ID options" + description: "Options for adding the id to .obs on the MuData object. Having a sample\ + \ \nid present in a requirement of several components for this pipeline.\n" + arguments: - type: "boolean" name: "--add_id_to_obs" description: "Add the value passed with --id to .obs." @@ -698,7 +698,7 @@ dependencies: repository: type: "vsh" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" - name: "neighbors/spatial_neighborhood_graph" repository: type: "local" @@ -707,17 +707,19 @@ dependencies: type: "local" - name: "dataflow/split_h5mu" repository: - type: "local" + type: "vsh" + repo: "openpipeline" + tag: "v4.0.3" - name: "workflows/multiomics/neighbors_leiden_umap" repository: type: "vsh" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -795,14 +797,14 @@ build_info: output: "target/nextflow/workflows/niche/nichecompass_leiden" executable: "target/nextflow/workflows/niche/nichecompass_leiden/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - - "target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu" + - "target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu" - "target/nextflow/neighbors/spatial_neighborhood_graph" - "target/nextflow/nichecompass/nichecompass" - - "target/nextflow/dataflow/split_h5mu" - - "target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap" + - "target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu" + - "target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap" package_config: name: "openpipeline_spatial" version: "niche-compass" @@ -815,7 +817,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/workflows/niche/nichecompass_leiden/main.nf b/target/nextflow/workflows/niche/nichecompass_leiden/main.nf index 3f13678..764f431 100644 --- a/target/nextflow/workflows/niche/nichecompass_leiden/main.nf +++ b/target/nextflow/workflows/niche/nichecompass_leiden/main.nf @@ -3182,6 +3182,18 @@ meta = [ "direction" : "input", "multiple" : false, "multiple_sep" : ";" + }, + { + "type" : "string", + "name" : "--var_input", + "description" : "Key in adata.var that indicates which features to include as input for the NicheCompass model.\nIf not provided, all features will be included.\n", + "default" : [ + "filter_with_hvg" + ], + "required" : false, + "direction" : "input", + "multiple" : false, + "multiple_sep" : ";" } ] }, @@ -3189,18 +3201,6 @@ meta = [ "name" : "Sample ID options", "description" : "Options for adding the id to .obs on the MuData object. Having a sample \nid present in a requirement of several components for this pipeline.\n", "arguments" : [ - { - "type" : "boolean", - "name" : "--include_sample_as_covariate", - "description" : "Whether to include the sample information as a categorical covariate for the \nNicheCompass model.\n", - "default" : [ - true - ], - "required" : false, - "direction" : "input", - "multiple" : false, - "multiple_sep" : ";" - }, { "type" : "boolean", "name" : "--add_id_to_obs", @@ -3822,7 +3822,7 @@ meta = [ "repository" : { "type" : "vsh", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } }, { @@ -3840,7 +3840,9 @@ meta = [ { "name" : "dataflow/split_h5mu", "repository" : { - "type" : "local" + "type" : "vsh", + "repo" : "openpipeline", + "tag" : "v4.0.3" } }, { @@ -3848,7 +3850,7 @@ meta = [ "repository" : { "type" : "vsh", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } } ], @@ -3857,7 +3859,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3948,7 +3950,7 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/niche/nichecompass_leiden", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3968,7 +3970,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", @@ -3991,11 +3993,11 @@ meta = [ // resolve dependencies dependencies (if any) meta["root_dir"] = getRootDir() -include { concatenate_h5mu } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/dataflow/concatenate_h5mu/main.nf" +include { concatenate_h5mu } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/concatenate_h5mu/main.nf" include { spatial_neighborhood_graph } from "${meta.resources_dir}/../../../../nextflow/neighbors/spatial_neighborhood_graph/main.nf" include { nichecompass } from "${meta.resources_dir}/../../../../nextflow/nichecompass/nichecompass/main.nf" -include { split_h5mu } from "${meta.resources_dir}/../../../../nextflow/dataflow/split_h5mu/main.nf" -include { neighbors_leiden_umap } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf" +include { split_h5mu } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/dataflow/split_h5mu/main.nf" +include { neighbors_leiden_umap } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/multiomics/neighbors_leiden_umap/main.nf" // inner workflow // user-provided Nextflow code @@ -4118,6 +4120,7 @@ workflow run_wf { "input": state.input, "input_gp_mask": state.input_gp_mask, "input_obs_covariates": state.input_obs_covariates, + "var_input": state.var_input, "modality": state.modality, "layer": state.layer, "min_genes_per_gp": state.min_genes_per_gp, diff --git a/target/nextflow/workflows/niche/nichecompass_leiden/nextflow_schema.json b/target/nextflow/workflows/niche/nichecompass_leiden/nextflow_schema.json index e34e852..7232ef4 100644 --- a/target/nextflow/workflows/niche/nichecompass_leiden/nextflow_schema.json +++ b/target/nextflow/workflows/niche/nichecompass_leiden/nextflow_schema.json @@ -61,6 +61,12 @@ "description": "Key in adata.obsm where spatial coordinates are stored", "help_text": "Type: `string`, multiple: `False`, default: `\"spatial\"`. ", "default": "spatial" + }, + "var_input": { + "type": "string", + "description": "Key in adata.var that indicates which features to include as input for the NicheCompass model.\nIf not provided, all features will be included.\n", + "help_text": "Type: `string`, multiple: `False`, default: `\"filter_with_hvg\"`. ", + "default": "filter_with_hvg" } } }, @@ -96,12 +102,6 @@ "type": "object", "description": "Options for adding the id to .obs on the MuData object. Having a sample \nid present in a requirement of several components for this pipeline.\n", "properties": { - "include_sample_as_covariate": { - "type": "boolean", - "description": "Whether to include the sample information as a categorical covariate for the \nNicheCompass model.\n", - "help_text": "Type: `boolean`, multiple: `False`, default: `true`. ", - "default": true - }, "add_id_to_obs": { "type": "boolean", "description": "Add the value passed with --id to .obs.", diff --git a/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml b/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml index defb6d0..e8dc83a 100644 --- a/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml +++ b/target/nextflow/workflows/qc/spatial_qc/.config.vsh.yaml @@ -304,12 +304,12 @@ dependencies: repository: type: "vsh" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" repositories: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" links: repository: "https://github.com/openpipelines-bio/openpipeline_spatial" docker_registry: "ghcr.io" @@ -387,10 +387,10 @@ build_info: output: "target/nextflow/workflows/qc/spatial_qc" executable: "target/nextflow/workflows/qc/spatial_qc/main.nf" viash_version: "0.9.4" - git_commit: "ad19aba349736848d99a9646870d8e7868eb6515" + git_commit: "87e62605aafd706d539bf2978ef47ede6fe41926" git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial" dependencies: - - "target/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc" + - "target/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc" package_config: name: "openpipeline_spatial" version: "niche-compass" @@ -403,7 +403,7 @@ package_config: - type: "vsh" name: "openpipeline" repo: "openpipeline" - tag: "v4.0.2" + tag: "v4.0.3" viash_version: "0.9.4" source: "src" target: "target" diff --git a/target/nextflow/workflows/qc/spatial_qc/main.nf b/target/nextflow/workflows/qc/spatial_qc/main.nf index e38e0e1..672c13a 100644 --- a/target/nextflow/workflows/qc/spatial_qc/main.nf +++ b/target/nextflow/workflows/qc/spatial_qc/main.nf @@ -3405,7 +3405,7 @@ meta = [ "repository" : { "type" : "vsh", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } } ], @@ -3414,7 +3414,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "links" : { @@ -3505,7 +3505,7 @@ meta = [ "engine" : "native", "output" : "/workdir/root/repo/target/nextflow/workflows/qc/spatial_qc", "viash_version" : "0.9.4", - "git_commit" : "ad19aba349736848d99a9646870d8e7868eb6515", + "git_commit" : "87e62605aafd706d539bf2978ef47ede6fe41926", "git_remote" : "https://github.com/openpipelines-bio/openpipeline_spatial" }, "package_config" : { @@ -3525,7 +3525,7 @@ meta = [ "type" : "vsh", "name" : "openpipeline", "repo" : "openpipeline", - "tag" : "v4.0.2" + "tag" : "v4.0.3" } ], "viash_version" : "0.9.4", @@ -3548,7 +3548,7 @@ meta = [ // resolve dependencies dependencies (if any) meta["root_dir"] = getRootDir() -include { qc as spatial_qc_workflow_viashalias } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.2/nextflow/workflows/qc/qc/main.nf" +include { qc as spatial_qc_workflow_viashalias } from "${meta.root_dir}/dependencies/vsh/vsh/openpipeline/v4.0.3/nextflow/workflows/qc/qc/main.nf" spatial_qc_workflow = spatial_qc_workflow_viashalias.run(key: "spatial_qc_workflow") // inner workflow