vsh/openpipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch fix-integration-tests with version dev (2dbe3b72)

Browse Source 
Build pipeline: vsh-ci-dev-k8tz4

Source commit: 2dbe3b7231

Source message: Fix pointers to test resources
This commit is contained in:
CI
2024-10-17 17:56:12 +00:00
commit cd0af18851
2125 changed files with 1018836 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

555
target/executable/query/cellxgene_census/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,555 @@
name: "cellxgene_census"
namespace: "query"
version: "dev"
authors:
- name: "Matthias Beyens"
roles:
- "maintainer"
- "author"
info:
role: "Contributor"
links:
github: "MatthiasBeyens"
orcid: "0000-0003-3304-0706"
email: "matthias.beyens@gmail.com"
linkedin: "mbeyens"
organizations:
- name: "Janssen Pharmaceuticals"
href: "https://www.janssen.com"
role: "Principal Scientist"
- name: "Dries De Maeyer"
roles:
- "author"
info:
role: "Core Team Member"
links:
email: "ddemaeyer@gmail.com"
github: "ddemaeyer"
linkedin: "dries-de-maeyer-b46a814"
organizations:
- name: "Janssen Pharmaceuticals"
href: "https://www.janssen.com"
role: "Principal Scientist"
- name: "Robrecht Cannoodt"
roles:
- "author"
info:
role: "Core Team Member"
links:
email: "robrecht@data-intuitive.com"
github: "rcannood"
orcid: "0000-0003-3641-729X"
linkedin: "robrechtcannoodt"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Data Science Engineer"
- name: "Open Problems"
href: "https://openproblems.bio"
role: "Core Member"
- name: "Kai Waldrant"
roles:
- "contributor"
info:
role: "Contributor"
links:
email: "kai@data-intuitive.com"
github: "KaiWaldrant"
orcid: "0009-0003-8555-1361"
linkedin: "kaiwaldrant"
organizations:
- name: "Data Intuitive"
href: "https://www.data-intuitive.com"
role: "Bioinformatician"
- name: "Open Problems"
href: "https://openproblems.bio"
role: "Contributor"
argument_groups:
- name: "Input database"
description: "Open CellxGene Census by version or URI."
arguments:
- type: "string"
name: "--input_uri"
description: "If specified, a URI containing the Census SOMA objects. If specified,\
\ will take precedence over the `--census_version` argument."
info: null
example:
- "s3://bucket/path"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--census_version"
description: "Which release of CellxGene census to use. Possible values are \"\
latest\", \"stable\", or the date of one of the releases (e.g. \"2023-07-25\"\
). For more information, check the documentation on [Census data releases](https://chanzuckerberg.github.io/cellxgene-census/cellxgene_census_docsite_data_release_info.html)."
info: null
example:
- "stable"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "boolean_true"
name: "--add_dataset_metadata"
description: "If true, the experiment metadata will be added to the cell metadata.\
\ More specifically: `collection_id`, `collection_name`, `collection_doi`, `dataset_title`."
info: null
direction: "input"
- name: "Cell query"
description: "Arguments related to the query."
arguments:
- type: "string"
name: "--species"
description: "The organism to query, usually one of `Homo sapiens` or `Mus musculus`."
info: null
example:
- "homo_sapiens"
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--obs_value_filter"
description: "Filter for selecting the `obs` metadata (i.e. cells). Value is a\
\ filter query written in the SOMA `value_filter` syntax."
info: null
example:
- "is_primary_data == True and cell_type_ontology_term_id in ['CL:0000136', 'CL:1000311',\
\ 'CL:0002616'] and suspension_type == 'cell'"
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Filter cells by grouping"
description: "Filter groups with fewer than X number of cells."
arguments:
- type: "string"
name: "--cell_filter_grouping"
description: "A subset of 'obs' columns by which to group the cells for filtering.\n\
Only groups surpassing or equal to the `--cell_filter_minimum_count`\nthreshold\
\ will be retained. Take care not to introduce a selection\nbias against cells\
\ with more fine-grained ontology annotations.\n"
info: null
example:
- "dataset_id"
- "tissue"
- "assay"
- "disease"
- "cell_type"
required: false
direction: "input"
multiple: true
multiple_sep: ";"
- type: "integer"
name: "--cell_filter_minimum_count"
description: "A minimum number of cells per group to retain. If `--cell_filter_grouping`\n\
is defined, this parameter should also be provided and vice versa.\n"
info: null
example:
- 100
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Count filtering"
description: "Arguments related to filtering cells and genes by counts."
arguments:
- type: "integer"
name: "--cell_filter_min_genes"
description: "Remove cells with less than this number of genes."
info: null
default:
- 50
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "integer"
name: "--cell_filter_min_counts"
description: "Remove cells with less than this number of counts."
info: null
default:
- 0
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "integer"
name: "--gene_filter_min_cells"
description: "Remove genes expressed in less than this number of cells."
info: null
default:
- 5
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "integer"
name: "--gene_filter_min_counts"
description: "Remove genes with less than this number of counts."
info: null
default:
- 0
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Outputs"
description: "Output arguments."
arguments:
- type: "file"
name: "--output"
description: "Output h5mu file."
info:
label: "CellxGene dataset"
summary: "A dataset queried from the CellxGene Census platform"
description: "The format of this file is derived from the [CELLxGENE schema\
\ v4.0.0](https://github.com/chanzuckerberg/single-cell-curation/blob/main/schema/4.0.0/schema.md).\n"
slots:
mod:
- name: "rna"
layers:
- type: "integer"
name: "counts"
description: "Raw counts"
required: true
obs:
- type: "string"
name: "dataset_id"
description: "Identifier for the dataset from which the cell data is derived,\
\ useful for tracking and referencing purposes."
required: false
- type: "string"
name: "assay"
description: "Type of assay used to generate the cell data, indicating\
\ the methodology or technique employed."
required: true
- type: "string"
name: "assay_ontology_term_id"
description: "Experimental Factor Ontology (`EFO:`) term identifier for\
\ the assay, providing a standardized reference to the assay type."
required: true
- type: "string"
name: "cell_type"
description: "Classification of the cell type based on its characteristics\
\ and function within the tissue or organism."
required: true
- type: "string"
name: "cell_type_ontology_term_id"
description: "Cell Ontology (`CL:`) term identifier for the cell type,\
\ offering a standardized reference to the specific cell classification."
required: true
- type: "string"
name: "development_stage"
description: "Stage of development of the organism or tissue from which\
\ the cell is derived, indicating its maturity or developmental phase."
required: true
- type: "string"
name: "development_stage_ontology_term_id"
description: "Ontology term identifier for the developmental stage, providing\
\ a standardized reference to the organism's developmental phase.\n\n\
If the organism is human (`organism_ontology_term_id == 'NCBITaxon:9606'`),\
\ then the Human Developmental Stages (`HsapDv:`) ontology is used.\
\ \nIf the organism is mouse (`organism_ontology_term_id == 'NCBITaxon:10090'`),\
\ then the Mouse Developmental Stages (`MmusDv:`) ontology is used.\n\
Otherwise, the Uberon (`UBERON:`) ontology is used.\n"
required: true
- type: "string"
name: "disease"
description: "Information on any disease or pathological condition associated\
\ with the cell or donor."
required: true
- type: "string"
name: "disease_ontology_term_id"
description: "Ontology term identifier for the disease, enabling standardized\
\ disease classification and referencing.\n\nMust be a term from the\
\ Mondo Disease Ontology (`MONDO:`) ontology term, or `PATO:0000461`\
\ from the Phenotype And Trait Ontology (`PATO:`).\n"
required: true
- type: "string"
name: "donor_id"
description: "Identifier for the donor from whom the cell sample is obtained."
required: true
- type: "boolean"
name: "is_primary_data"
description: "Indicates whether the data is primary (directly obtained\
\ from experiments) or has been computationally derived from other primary\
\ data."
required: true
- type: "string"
name: "organism"
description: "Organism from which the cell sample is obtained."
required: true
- type: "string"
name: "organism_ontology_term_id"
description: "Ontology term identifier for the organism, providing a standardized\
\ reference for the organism.\n\nMust be a term from the NCBI Taxonomy\
\ Ontology (`NCBITaxon:`) which is a child of `NCBITaxon:33208`.\n"
required: true
- type: "string"
name: "self_reported_ethnicity"
description: "Ethnicity of the donor as self-reported, relevant for studies\
\ considering genetic diversity and population-specific traits."
required: true
- type: "string"
name: "self_reported_ethnicity_ontology_term_id"
description: "Ontology term identifier for the self-reported ethnicity,\
\ providing a standardized reference for ethnic classifications.\n\n\
If the organism is human (`organism_ontology_term_id == 'NCBITaxon:9606'`),\
\ then the Human Ancestry Ontology (`HANCESTRO:`) is used.\n"
required: true
- type: "string"
name: "sex"
description: "Biological sex of the donor or source organism, crucial\
\ for studies involving sex-specific traits or conditions."
required: true
- type: "string"
name: "sex_ontology_term_id"
description: "Ontology term identifier for the biological sex, ensuring\
\ standardized classification of sex. Only `PATO:0000383`, `PATO:0000384`\
\ and `PATO:0001340` are allowed."
required: true
- type: "string"
name: "suspension_type"
description: "Type of suspension or medium in which the cells were stored\
\ or processed, important for understanding cell handling and conditions."
required: true
- type: "string"
name: "tissue"
description: "Specific tissue from which the cells were derived, key for\
\ context and specificity in cell studies."
required: true
- type: "string"
name: "tissue_ontology_term_id"
description: "Ontology term identifier for the tissue, providing a standardized\
\ reference for the tissue type.\n\nFor organoid or tissue samples,\
\ the Uber-anatomy ontology (`UBERON:`) is used. The term ids must be\
\ a child term of `UBERON:0001062` (anatomical entity).\nFor cell cultures,\
\ the Cell Ontology (`CL:`) is used. The term ids cannot be `CL:0000255`,\
\ `CL:0000257` or `CL:0000548`.\n"
required: true
- type: "string"
name: "tissue_general"
description: "General category or classification of the tissue, useful\
\ for broader grouping and comparison of cell data."
required: true
- type: "string"
name: "tissue_general_ontology_term_id"
description: "Ontology term identifier for the general tissue category,\
\ aiding in standardizing and grouping tissue types.\n\nFor organoid\
\ or tissue samples, the Uber-anatomy ontology (`UBERON:`) is used.\
\ The term ids must be a child term of `UBERON:0001062` (anatomical\
\ entity).\nFor cell cultures, the Cell Ontology (`CL:`) is used. The\
\ term ids cannot be `CL:0000255`, `CL:0000257` or `CL:0000548`.\n"
required: true
- type: "integer"
name: "soma_joinid"
description: "If the dataset was retrieved from CELLxGENE census, this\
\ is a unique identifier for the cell."
required: true
var:
- type: "string"
name: "feature_id"
description: "Unique identifier for the feature, usually a ENSEMBL gene\
\ id."
required: true
- type: "string"
name: "feature_name"
description: "A human-readable name for the feature, usually a gene symbol."
required: true
- type: "integer"
name: "soma_joinid"
description: "If the dataset was retrieved from CELLxGENE census, this\
\ is a unique identifier for the feature."
required: true
example:
- "output.h5mu"
must_exist: true
create_parent: true
required: true
direction: "output"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_compression"
info: null
example:
- "gzip"
required: false
choices:
- "gzip"
- "lzf"
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--output_modality"
description: "Which modality to store the output in."
info: null
default:
- "rna"
required: false
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "setup_logger.py"
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Query cells from a CellxGene Census or custom TileDBSoma object.\nAside\
\ from fetching the cells' RNA counts (`.X`), cell metadata\n(`.obs`) and gene metadata\
\ (`.var`), this component also fetches\nthe dataset metadata and joins it into\
\ the cell metadata.\n"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "openpipelinetestutils"
dest: "openpipelinetestutils"
info: null
status: "enabled"
links:
repository: "https://github.com/openpipelines-bio/openpipeline"
docker_registry: "ghcr.io"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
label:
- "highmem"
- "midcpu"
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.11"
target_registry: "images.viash-hub.com"
target_tag: "dev"
namespace_separator: "/"
setup:
- type: "python"
user: false
packages:
- "anndata==0.10.8"
- "mudata~=0.2.4"
- "pandas!=2.1.2"
- "numpy<2.0.0"
- "scanpy~=1.9.6"
- "cellxgene-census"
upgrade: true
test_setup:
- type: "python"
user: false
packages:
- "viashpy==0.8.0"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/query/cellxgene_census/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/query/cellxgene_census"
executable: "target/executable/query/cellxgene_census/cellxgene_census"
viash_version: "0.9.0"
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
git_tag: "0.2.0-1926-g2dbe3b72"
package_config:
name: "openpipeline"
version: "dev"
info:
test_resources:
- type: "s3"
path: "s3://openpipelines-data"
dest: "resources_test"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
\ := 'includeConfig(\"nextflow_labels.config\")'"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'dev'"
organization: "vsh"
links:
repository: "https://github.com/openpipelines-bio/openpipeline"
docker_registry: "ghcr.io"
homepage: "https://openpipelines.bio"
documentation: "https://openpipelines.bio/fundamentals"
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"

1537
target/executable/query/cellxgene_census/cellxgene_census Executable file
View File

File diff suppressed because it is too large Load Diff

42
target/executable/query/cellxgene_census/nextflow_labels.config Normal file
View File

@@ -0,0 +1,42 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

12
target/executable/query/cellxgene_census/setup_logger.py Normal file
View File

@@ -0,0 +1,12 @@
def setup_logger():
import logging
from sys import stdout
logger = logging.getLogger()
logger.setLevel(logging.INFO)
console_handler = logging.StreamHandler(stdout)
logFormatter = logging.Formatter("%(asctime)s %(levelname)-8s %(message)s")
console_handler.setFormatter(logFormatter)
logger.addHandler(console_handler)
return logger