vsh/openpipeline_spatial
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch allow-compressed-output-bundles with version allow-compressed-output-bundles (b1bfccb)

Browse Source 
Build pipeline: openpipelines-bio.openpipeline-spatial.allow-compressed-ou2g89k

Source commit: b1bfccbd0e

Source message: wip
This commit is contained in:
CI
2025-08-27 11:21:44 +00:00
commit 321cb96cfa
424 changed files with 229459 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

327
target/executable/convert/from_cells2stats_to_h5mu/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,327 @@
name: "from_cells2stats_to_h5mu"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Inputs"
arguments:
- type: "file"
name: "--input"
description: "Path to the cells2stats output bundle. \nExpected folder structure\
\ (showing required files only):\n├── Cytoprofiling/\n│ └── Instrument/\n│\
\ └── RawCellStats.parquet\n└── Panel.json\n"
info: null
example:
- "path/to/aviti_output"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Outputs"
arguments:
- type: "file"
name: "--output"
description: "Output H5MU file path."
info: null
example:
- "output.h5mu"
must_exist: true
create_parent: true
required: true
direction: "output"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_compression"
description: "Compression format to use for the output AnnData and/or Mudata objects.\n\
By default no compression is applied.\n"
info: null
example:
- "gzip"
required: false
choices:
- "gzip"
- "lzf"
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Options"
arguments:
- type: "string"
name: "--modality"
description: "The modality to which the processed data will be written to in the\
\ H5MU file."
info: null
default:
- "rna"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_coordinates"
description: "Key name to store the spatial coordinates (in pixels) in obsm.\n\
If present, spatial coordinates in micrometers will be stored under {obsm_coordinates}_um.\n\
The column names will be stored in uns.\n"
info: null
default:
- "spatial"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--layer_nuclear_counts"
description: "Name for nuclear counts layer. If specified, nuclear count data\
\ \nwill be stored as a separate layer in the AnnData object.\n"
info: null
example:
- "nuclear_counts"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_cell_paint"
description: "Key name for storing Cell Paint target intensities in obsm. \nIf\
\ provided, Cell Paint target intensity data will be stored as a separate matrix\
\ in the obsm field.\nThe column names will be stored in uns.\n"
info: null
example:
- "cell_paint"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_cell_paint_nuclear"
description: "Key name for storing Nuclear Cell Paint target intensities in obsm.\n\
If provided, Nuclear Cell Paint target intensity data will be stored as a separate\
\ matrix in the obsm field.\nThe column names will be stored in uns.\n"
info: null
example:
- "cell_paint_nuclear"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_cell_profiler"
description: "Key name for storing CellProfiler morphology metrics in obsm.\n\
If provided, CellProfiler morphology metrics will be stored as a separate matrix\
\ in the obsm field.\nThe column names will be stored in uns.\n"
info: null
example:
- "cell_profiler"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_unassigned_targets"
description: "Key name for storing any unassigned target data in obsm.\nIf provided,\
\ unassigned target data will be stored as a separate matrix in the obsm field.\n\
The column names will be stored in uns.\n"
info: null
example:
- "cell_profiler"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "unzip_archived_folder.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Convert spatial data resulting from Aviti Teton sequencers that have\
\ been processed by the Element Biosciences cells2stats workflow to H5MU format.\n\
\nThis component processes cells2stats count matrices to create a standardized H5MU\
\ file for downstream analysis.\n\nThe component reads:\n- Parquet file containing\
\ the count matrix and metadata\n- Panel.json with target and batch information\n\
\nAnd outputs an H5MU file with:\n- Count data as the main .X matrix\n- Spatial\
\ coordinates in obsm\n- Cell Paint intensities in obsm (optional)\n- Nuclear count\
\ data as a layer (optional)\n- CellProfiler morphology metrics in obsm (optional)\n\
- Unassigned targets in obsm (optional)\n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "aviti"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "lowcpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.13-slim"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "anndata~=0.11.1"
- "mudata~=0.3.1"
- "pyarrow"
script:
- "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\
nelse: exit(1)\")"
upgrade: true
test_setup:
- type: "apt"
packages:
- "zip"
interactive: false
- type: "python"
user: false
packages:
- "viashpy==0.9.0"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_cells2stats_to_h5mu/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_cells2stats_to_h5mu"
executable: "target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1603
target/executable/convert/from_cells2stats_to_h5mu/from_cells2stats_to_h5mu Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_cells2stats_to_h5mu/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/convert/from_cells2stats_to_h5mu/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger

53
target/executable/convert/from_cells2stats_to_h5mu/unzip_archived_folder.py Normal file
View File

@@ -0,0 +1,53 @@
import fnmatch
import zipfile
import tempfile
from pathlib import Path
from typing import Union
def unzip_archived_folder(
archived_folder: Union[str, Path]
) -> Union[str, Path]:
"""
Extracts a ZIP archive to a temporary directory and returns the path to the extracted folder.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
Returns:
extracted_path (Union[str, Path]): Path to the extracted folder inside the temporary directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(archived_folder, "r") as archive:
archive.extractall(temp_dir)
return temp_dir
def extract_selected_files_from_zip(
zip_path: Union[str, Path],
members: list[Union[str, Path]]
) -> Union[str, Path]:
"""
Extracts selected files (supports glob patterns) from a ZIP archive to a temporary directory.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
members (list[str]): List of file paths within the archive to extract.
Returns:
Path: Path to the extraction directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(zip_path, "r") as archive:
all_files = archive.namelist()
selected = set()
for pattern in members:
selected.update(fnmatch.filter(all_files, str(pattern)))
for member in selected:
archive.extract(member, temp_dir)
return temp_dir

262
target/executable/convert/from_cosmx_to_h5mu/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,262 @@
name: "from_cosmx_to_h5mu"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
- name: "Weiwei Schultz"
roles:
- "contributor"
info:
role: "Contributor"
organizations:
- name: "Janssen R&D US"
role: "Associate Director Data Sciences"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input folder. Must contain the output from a NanoString CosMx run."
info: null
example:
- "cosmx_data"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--modality"
info: null
default:
- "rna"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "The output h5mu file."
info: null
example:
- "output.h5mu"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_compression"
info: null
example:
- "gzip"
required: false
choices:
- "gzip"
- "lzf"
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "unzip_archived_folder.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Converts the output from NanoString experiment into a MuData objcet.\n\
\ - `<dataset_id>_exprMat_file.csv`: File containing the counts.\n - `<dataset_id>`_metadata_file:\
\ File containing the spatial coordinates and additional cell-level metadata.\n\
\ - `<dataset_id>_fov_file.csv`: File containing the coordinates of all the fields\
\ of view.\nIn addition to reading the regular Nanostring output, it loads CellComposite\
\ and CellLabels directories, if present,\ncontaining the images.\n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "Lung5_Rep2_tiny"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.12-slim"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "anndata~=0.11.1"
- "mudata~=0.3.1"
- "spatialdata~=0.4.1rc"
- "squidpy~=1.6.5"
script:
- "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\
nelse: exit(1)\")"
upgrade: true
test_setup:
- type: "apt"
packages:
- "git"
interactive: false
- type: "python"
user: false
packages:
- "viashpy==0.9.0"
github:
- "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils"
upgrade: true
- type: "apt"
packages:
- "zip"
interactive: false
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_cosmx_to_h5mu/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_cosmx_to_h5mu"
executable: "target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1249
target/executable/convert/from_cosmx_to_h5mu/from_cosmx_to_h5mu Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_cosmx_to_h5mu/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/convert/from_cosmx_to_h5mu/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger

53
target/executable/convert/from_cosmx_to_h5mu/unzip_archived_folder.py Normal file
View File

@@ -0,0 +1,53 @@
import fnmatch
import zipfile
import tempfile
from pathlib import Path
from typing import Union
def unzip_archived_folder(
archived_folder: Union[str, Path]
) -> Union[str, Path]:
"""
Extracts a ZIP archive to a temporary directory and returns the path to the extracted folder.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
Returns:
extracted_path (Union[str, Path]): Path to the extracted folder inside the temporary directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(archived_folder, "r") as archive:
archive.extractall(temp_dir)
return temp_dir
def extract_selected_files_from_zip(
zip_path: Union[str, Path],
members: list[Union[str, Path]]
) -> Union[str, Path]:
"""
Extracts selected files (supports glob patterns) from a ZIP archive to a temporary directory.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
members (list[str]): List of file paths within the archive to extract.
Returns:
Path: Path to the extraction directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(zip_path, "r") as archive:
all_files = archive.namelist()
selected = set()
for pattern in members:
selected.update(fnmatch.filter(all_files, str(pattern)))
for member in selected:
archive.extract(member, temp_dir)
return temp_dir

263
target/executable/convert/from_cosmx_to_spatialexperiment/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,263 @@
name: "from_cosmx_to_spatialexperiment"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "author"
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input CosMx directory"
info: null
example:
- "path/to/cosmx_bundle"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--add_tx_path"
description: "Whether to add parquet paths to the metadata. \nIf True, `*_tx_file.csv`\
\ file will be converted to .parquet and added to the metadata. \n"
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--add_polygon_path"
description: "Whether to add polygon path to the metadata.\nIf True, `*_polygons.csv`\
\ file will be converted to .parquet and be added to the metadata.\n"
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--add_fov_positions"
description: "Whether to add fov positions to the metadata.\nIf True, `fov_positions_file.csv`\
\ will be added to the metadata.\n"
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--alternative_experiment_features"
description: "Feature names containing these strings will be moved to altExps(sxe)\
\ slots as separate SpatialExperiment objects."
info: null
default:
- "NegPrb"
- "Negative"
- "SystemControl"
- "FalseCode"
required: false
direction: "input"
multiple: true
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "Output SpatialExperiment file"
info: null
example:
- "output.rds"
must_exist: true
create_parent: true
required: true
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "r_script"
path: "script.R"
is_executable: true
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Creates a SpatialExperiment object from the downloaded unzipped CosMx\
\ directory for Nanostring\nCosMx spatial gene expression data, and saves it as\
\ a SpatialExperiment object.\nThe constructor assumes the downloaded unzipped CosMx\
\ Folder has the following structure:\n\nMandatory files\n· | — *_exprMat_file.csv\n\
· | — *_metadata_file.csv\nOptional files, by default added to the metadata() as\
\ a list of paths (will be converted to parquet):\n· | —*_fov_positions_file.csv\n\
· | — *_tx_file.csv\n· | — *_polygons.csv\n"
test_resources:
- type: "r_script"
path: "test.R"
is_executable: true
- type: "file"
path: "Lung5_Rep2_tiny"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "rocker/r2u:24.04"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "libhdf5-dev"
- "libgeos-dev"
interactive: false
- type: "r"
bioc:
- "SpatialExperimentIO"
bioc_force_install: false
warnings_as_errors: true
test_setup:
- type: "r"
cran:
- "testthat"
bioc_force_install: false
warnings_as_errors: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_cosmx_to_spatialexperiment/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_cosmx_to_spatialexperiment"
executable: "target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1283
target/executable/convert/from_cosmx_to_spatialexperiment/from_cosmx_to_spatialexperiment Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_cosmx_to_spatialexperiment/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

255
target/executable/convert/from_h5mu_to_spatialexperiment/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,255 @@
name: "from_h5mu_to_spatialexperiment"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "author"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input h5mu file"
info: null
example:
- "input.h5mu"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--modality"
description: "Name of the modality to be converted."
info: null
default:
- "rna"
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_spatial_coordinates"
description: "Key in the .obsm field that contains the spatial coordinates. \n\
Will be mapped to spatialCoords in the SpatialExperiment object.\n"
info: null
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "Output SpatialExperiment file"
info: null
example:
- "output.rds"
must_exist: true
create_parent: true
required: true
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "r_script"
path: "script.R"
is_executable: true
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Converts an h5mu file into a SpatialExperiment object.\n"
test_resources:
- type: "r_script"
path: "test.R"
is_executable: true
- type: "file"
path: "aviti_teton_tiny.h5mu"
- type: "file"
path: "Lung5_Rep2_tiny.h5mu"
- type: "file"
path: "xenium_tiny.h5mu"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "rocker/r2u:22.04"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "libhdf5-dev"
- "libgeos-dev"
interactive: false
- type: "r"
cran:
- "hdf5r"
- "SpatialExperiment"
github:
- "scverse/anndataR@36f3caad9a7f360165c1510bbe0c62657580415a"
bioc_force_install: false
warnings_as_errors: true
test_setup:
- type: "docker"
env:
- "RETICULATE_PYTHON=/usr/bin/python"
- type: "apt"
packages:
- "python3"
- "python3-pip"
- "python3-dev"
- "python-is-python3"
interactive: false
- type: "r"
cran:
- "reticulate"
- "testthat"
bioc_force_install: false
warnings_as_errors: true
- type: "python"
user: false
packages:
- "anndata~=0.11.1"
- "mudata~=0.3.1"
script:
- "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\
nelse: exit(1)\")"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_h5mu_to_spatialexperiment/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_h5mu_to_spatialexperiment"
executable: "target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1278
target/executable/convert/from_h5mu_to_spatialexperiment/from_h5mu_to_spatialexperiment Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_h5mu_to_spatialexperiment/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

251
target/executable/convert/from_spatialdata_to_h5mu/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,251 @@
name: "from_spatialdata_to_h5mu"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
- name: "Weiwei Schultz"
roles:
- "contributor"
info:
role: "Contributor"
organizations:
- name: "Janssen R&D US"
role: "Associate Director Data Sciences"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input zarr folder where the SpatialData object is stored."
info: null
example:
- "input.zarr"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--modality"
info: null
default:
- "rna"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "The output h5mu file."
info: null
example:
- "output.h5mu"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_compression"
info: null
example:
- "gzip"
required: false
choices:
- "gzip"
- "lzf"
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Reads in the Tables field stored in a SpatialData object and converts\
\ it to an h5mu file.\n \n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "xenium_tiny.zarr"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.12-slim"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "anndata~=0.11.1"
- "mudata~=0.3.1"
- "spatialdata~=0.4.1rc"
script:
- "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\
nelse: exit(1)\")"
upgrade: true
test_setup:
- type: "apt"
packages:
- "git"
interactive: false
- type: "python"
user: false
packages:
- "viashpy==0.9.0"
github:
- "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_spatialdata_to_h5mu/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_spatialdata_to_h5mu"
executable: "target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1195
target/executable/convert/from_spatialdata_to_h5mu/from_spatialdata_to_h5mu Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_spatialdata_to_h5mu/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/convert/from_spatialdata_to_h5mu/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger

270
target/executable/convert/from_xenium_to_h5mu/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,270 @@
name: "from_xenium_to_h5mu"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input folder. Must contain the output from a Xenium run."
info: null
example:
- "xenium_output_bundle"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "Output .h5mu file."
info: null
example:
- "xenium.h5mu"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obsm_coordinates"
description: "Name of the .obsm slot under which to store the cell centroid coordinates."
info: null
default:
- "spatial"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--uns_experiment"
description: "Name of the .uns slot under which to store the Xenium experiment\
\ specifications."
info: null
default:
- "xenium_experiment"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--uns_metrics"
description: "Name of the .uns slot under which to store the summary QC metrics."
info: null
default:
- "xenium_metrics"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_compression"
description: "Compression format to use for the output AnnData and/or Mudata objects.\n\
By default no compression is applied.\n"
info: null
example:
- "gzip"
required: false
choices:
- "gzip"
- "lzf"
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "unzip_archived_folder.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Converts the output from Xenium to a single .h5mu file, where the count\
\ matrix is written to the `rna` modality.\nThe following files are expected to\
\ be present in the Xenium output bundle:\n├── cell_feature_matrix.h5\n├── cells.parquet\n\
├── experiment.xenium\n└── metrics_summary.csv\n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "xenium_tiny"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.12-slim"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "anndata~=0.11.1"
- "mudata~=0.3.1"
- "scanpy~=1.10.4"
- "pyarrow"
script:
- "exec(\"try:\\n import awkward\\nexcept ModuleNotFoundError:\\n exit(0)\\\
nelse: exit(1)\")"
upgrade: true
test_setup:
- type: "apt"
packages:
- "zip"
interactive: false
- type: "python"
user: false
packages:
- "viashpy==0.9.0"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_xenium_to_h5mu/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_xenium_to_h5mu"
executable: "target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1311
target/executable/convert/from_xenium_to_h5mu/from_xenium_to_h5mu Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_xenium_to_h5mu/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/convert/from_xenium_to_h5mu/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger

53
target/executable/convert/from_xenium_to_h5mu/unzip_archived_folder.py Normal file
View File

@@ -0,0 +1,53 @@
import fnmatch
import zipfile
import tempfile
from pathlib import Path
from typing import Union
def unzip_archived_folder(
archived_folder: Union[str, Path]
) -> Union[str, Path]:
"""
Extracts a ZIP archive to a temporary directory and returns the path to the extracted folder.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
Returns:
extracted_path (Union[str, Path]): Path to the extracted folder inside the temporary directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(archived_folder, "r") as archive:
archive.extractall(temp_dir)
return temp_dir
def extract_selected_files_from_zip(
zip_path: Union[str, Path],
members: list[Union[str, Path]]
) -> Union[str, Path]:
"""
Extracts selected files (supports glob patterns) from a ZIP archive to a temporary directory.
Args:
zip_path (Union[str, Path]): Path to the ZIP archive.
members (list[str]): List of file paths within the archive to extract.
Returns:
Path: Path to the extraction directory.
"""
temp_dir = Path(tempfile.TemporaryDirectory().name)
with zipfile.ZipFile(zip_path, "r") as archive:
all_files = archive.namelist()
selected = set()
for pattern in members:
selected.update(fnmatch.filter(all_files, str(pattern)))
for member in selected:
archive.extract(member, temp_dir)
return temp_dir

345
target/executable/convert/from_xenium_to_spatialdata/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,345 @@
name: "from_xenium_to_spatialdata"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
- name: "Weiwei Schultz"
roles:
- "contributor"
info:
role: "Contributor"
organizations:
- name: "Janssen R&D US"
role: "Associate Director Data Sciences"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input folder. Must contain the output from a xenium run."
info: null
example:
- "xenium_data"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "Zarr directory where the SpatialData object will be stored"
info: null
example:
- "xenium_data.zarr"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--cells_boundaries"
description: "Whether to read cell boundaries (polygons)."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--nucleus_boundaries"
description: "Whether to read nucleus boundaries (polygons)."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean_true"
name: "--cells_as_circles"
description: "Whether to read cells also as circles (the center and the radius\
\ of each circle is computed from the corresponding labels cell)."
info: null
direction: "input"
- type: "boolean"
name: "--cells_labels"
description: "Whether to read cell labels (raster). The polygonal version of the\
\ cell labels are simplified for visualization purposes, and using the raster\
\ version is recommended for analysis."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--transcripts"
description: "Whether to read transcripts."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--nucleus_labels"
description: "Whether to read nucleus labels (raster). The polygonal version of\
\ the nucleus labels are simplified for visualization purposes, and using the\
\ raster version is recommended for analysis."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--morphology_mip"
description: "Whether to read the morphology mip image (available in versions\
\ < 2.0.0)."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--morphology_focus"
description: "Whether to read the morphology focus image."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--aligned_images"
description: "Whether to also parse, when available, additional H&E or IF aligned\
\ images. For more control over the aligned images being read, in particular,\
\ to specify the axes of the aligned images, please set this parameter to False\
\ and use the xenium_aligned_image function directly."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--cells_table"
description: "Whether to read the cell annotations in the AnnData table."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "integer"
name: "--n_jobs"
info: null
default:
- 1
required: false
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Converts the output from 10X Genomics Xenium dataset into a SpatialData\
\ objcet.\nBy default, the following files will be converted:\n - `experiment.xenium`:\
\ File containing specifications.\n - `nucleus_boundaries.parquet`: Polygons of\
\ nucleus boundaries.\n - `cell_boundaries.parquet`: Polygons of cell boundaries.\n\
\ - `transcripts.parquet`: File containing transcripts.\n - `cell_feature_matrix.h5`:\
\ File containing cell feature matrix.\n - `cells.parquet`: File containing cell\
\ metadata.\n - `morphology_mip.ome.tif`: File containing morphology mip.\n -\
\ `morphology_focus.ome.tif`: File containing morphology focus.\n \n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "xenium_tiny"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.12-slim"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "spatialdata~=0.4.1rc"
- "spatialdata-io~=0.2.0"
upgrade: true
test_setup:
- type: "apt"
packages:
- "git"
interactive: false
- type: "python"
user: false
packages:
- "viashpy==0.9.0"
github:
- "openpipelines-bio/core#subdirectory=packages/python/openpipeline_testutils"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_xenium_to_spatialdata/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_xenium_to_spatialdata"
executable: "target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1449
target/executable/convert/from_xenium_to_spatialdata/from_xenium_to_spatialdata Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_xenium_to_spatialdata/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/convert/from_xenium_to_spatialdata/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger

253
target/executable/convert/from_xenium_to_spatialexperiment/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,253 @@
name: "from_xenium_to_spatialexperiment"
namespace: "convert"
version: "allow-compressed-output-bundles"
authors:
- name: "Dorien Roosen"
roles:
- "author"
- "maintainer"
info:
role: "Core Team Member"
links:
email: "dorien@data-intuitive.com"
github: "dorien-er"
linkedin: "dorien-roosen"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Scientist"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
alternatives:
- "-i"
description: "Input Xenium Output Bundle"
info: null
example:
- "path/to/xenium_bundle"
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--add_experiment_xenium"
description: "Whether to add xenium.experiment parameters to the metadata."
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean"
name: "--add_parquet_paths"
description: "Whether to add parquet paths to the metadata. \nIf True, `transcripts.parquet`,\
\ `cell_boundaries.parquet`, `nucleus_boundaries.parquet` will be added to the\
\ metadata. \n"
info: null
default:
- true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--alternative_experiment_features"
description: "Feature names containing these strings will be moved to altExps(sxe)\
\ slots as separate SpatialExperiment objects."
info: null
default:
- "NegControlProbe"
- "UnassignedCodeword"
- "NegControlCodeword"
- "antisense"
- "BLANK"
required: false
direction: "input"
multiple: true
multiple_sep: ";"
- type: "file"
name: "--output"
alternatives:
- "-o"
description: "Output SpatialExperiment file"
info: null
example:
- "output.rds"
must_exist: true
create_parent: true
required: true
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "r_script"
path: "script.R"
is_executable: true
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Creates a SpatialExperiment object from the downloaded unzipped Xenium\
\ Output Bundle directory\nfor 10x Genomics Xenium spatial gene expression data,\
\ and saves it as a SpatialExperiment object.\nThe constructor assumes the downloaded\
\ unzipped Xenium Output Bundle has the following structure:\n\nMandatory files\n\
· | — cell_feature_matrix.h5\n· | — cells.parquet\nOptional files, by default added\
\ to the metadata() as a list of paths (will be converted to parquet):\n· | — transcripts.parquet\n\
· | — cell_boundaries.parquet\n· | — nucleus_boundaries.parquet\n· | — experiment.xenium\n"
test_resources:
- type: "r_script"
path: "test.R"
is_executable: true
- type: "file"
path: "xenium_tiny"
info: null
status: "enabled"
scope:
image: "public"
target: "public"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "lowmem"
- "singlecpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "rocker/r2u:24.04"
target_registry: "images.viash-hub.com"
target_tag: "allow-compressed-output-bundles"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "libhdf5-dev"
- "libgeos-dev"
interactive: false
- type: "r"
bioc:
- "SpatialExperimentIO"
bioc_force_install: false
warnings_as_errors: true
test_setup:
- type: "r"
cran:
- "testthat"
bioc_force_install: false
warnings_as_errors: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/convert/from_xenium_to_spatialexperiment/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/convert/from_xenium_to_spatialexperiment"
executable: "target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment"
viash_version: "0.9.4"
git_commit: "b1bfccbd0edea3f1692c489d0f174f7e494a7341"
git_remote: "https://github.com/openpipelines-bio/openpipeline_spatial"
package_config:
name: "openpipeline_spatial"
version: "allow-compressed-output-bundles"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-bio/openpipeline_spatial/resources_test"
dest: "resources_test"
repositories:
- type: "vsh"
name: "openpipeline"
repo: "openpipeline"
tag: "v3.0.0"
viash_version: "0.9.4"
source: "src"
target: "target"
config_mods:
- ".resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
)'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'allow-compressed-output-bundles'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline_spatial"
docker_registry: "ghcr.io"

1249
target/executable/convert/from_xenium_to_spatialexperiment/from_xenium_to_spatialexperiment Executable file
View File

File diff suppressed because it is too large Load Diff

68
target/executable/convert/from_xenium_to_spatialexperiment/nextflow_labels.config Normal file
View File

@@ -0,0 +1,68 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// CPU resources
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
// Memory resources
withLabel: lowmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
// Disk space
// Nextflow apparently can't handle empty directives, i.e.
// withLabel: lowdisk {}
// so for that reason we have to add a dummy directive
withLabel: lowdisk {
dummyDirective = "dummyValue"
}
withLabel: middisk {
dummyDirective = "dummyValue"
}
withLabel: highdisk {
dummyDirective = "dummyValue"
}
withLabel: veryhighdisk {
dummyDirective = "dummyValue"
}
// NOTE: The above labels intentionally do not have an effect by default.
// The user should set the disk space requirements by adding the following
// to the compute environment:
//
// withLabel: lowdisk { disk = { 20.GB * task.attempt } }
// withLabel: middisk { disk = { 100.GB * task.attempt } }
// withLabel: highdisk { disk = { 200.GB * task.attempt } }
// withLabel: veryhighdisk { disk = { 500.GB * task.attempt } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}