Build branch main with version main (b98f636)

Build pipeline: viash-hub.htrnaseq.main-48gzk

Source commit: b98f6367d6

Source message: Add BAM statistics calculations on pool and well level (#6)
This commit is contained in:
CI
2024-08-29 12:41:46 +00:00
parent 044a3af7a9
commit bde35f120c
43 changed files with 12072 additions and 43 deletions

View File

@@ -236,7 +236,7 @@ build_info:
output: "target/executable/parallel_map"
executable: "target/executable/parallel_map/parallel_map"
viash_version: "0.9.0-RC7"
git_commit: "21831c2104098ecce57aa9b372e49f865296cc48"
git_commit: "b98f6367d672368af134843711a46d3b53717187"
git_remote: "https://github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"

View File

@@ -516,9 +516,9 @@ RUN wget -O $STAR_TARGET $STAR_SOURCE && \
rm $STAR_TARGET && rm -rf /tmp/STAR_$STAR_VERSION
LABEL org.opencontainers.image.description="Companion container for running component parallel_map"
LABEL org.opencontainers.image.created="2024-08-29T07:58:04Z"
LABEL org.opencontainers.image.created="2024-08-29T12:31:01Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
LABEL org.opencontainers.image.revision="21831c2104098ecce57aa9b372e49f865296cc48"
LABEL org.opencontainers.image.revision="b98f6367d672368af134843711a46d3b53717187"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

View File

@@ -0,0 +1,186 @@
name: "generate_pool_statistics"
namespace: "stats"
version: "main"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--nrReadsNrGenesPerChrom"
description: "Path to an output file that contains a .tsv formatted table describing\n\
per chromosome the number of reads that were mapped to that chromosome (NumberOfReads\n\
column) and the number of genes on that chromosome that had at least one\nread\
\ mapped to it (NumberOfGenes).\n"
info: null
default:
- "processedBamFile_well1.tsv"
- "processedBamfile_well2.tsv"
must_exist: true
create_parent: true
required: false
direction: "input"
multiple: true
multiple_sep: ";"
- type: "file"
name: "--nrReadsNrGenesPerChromPool"
description: "Pivot table in tsv format of the combined input nrReadsNrGenesPerChrom\
\ files. Describes\nper chromosome (as columns) the number of reads, as well\
\ as the total number \nof reads per cell barcode and the percentage of nuclear,\
\ ERCC and mitochondrial\nreads.\n"
info: null
example:
- "nrReadsNrGenesPerChrom.txt"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
info: null
status: "enabled"
requirements:
commands:
- "ps"
license: "MIT"
links:
repository: "https://github.com/viash-hub/htrnaseq"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "python:3.11-slim"
target_registry: "images.viash-hub.com"
target_tag: "main"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "pandas"
upgrade: true
test_setup:
- type: "python"
user: false
packages:
- "viashpy"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/stats/generate_pool_statistics/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/stats/generate_pool_statistics"
executable: "target/executable/stats/generate_pool_statistics/generate_pool_statistics"
viash_version: "0.9.0-RC7"
git_commit: "b98f6367d672368af134843711a46d3b53717187"
git_remote: "https://github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\
\ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\
\ dest: 'nextflow_labels.config'}\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"
keywords:
- "bioinformatics"
- "sequence"
- "high-throughput"
- "mapping"
- "counting"
- "pipeline"
license: "MIT"
organization: "vsh"
links:
repository: "https://github.com/viash-hub/htrnaseq"
issue_tracker: "https://github.com/viash-hub/htrnaseq/issues"

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,43 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}

View File

@@ -0,0 +1,257 @@
name: "generate_well_statistics"
namespace: "stats"
version: "main"
argument_groups:
- name: "Arguments"
arguments:
- type: "file"
name: "--input"
description: "The .bam file as returned by the mapping tool STAR."
info: null
example:
- "input.bam"
must_exist: true
create_parent: true
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "string"
name: "--barcode"
description: "The barcode for the well that is being processed. Is only used to\
\ add a metadata\ncolumn to all output files.\n"
info: null
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--processedBAMFile"
description: "Path to a .tsv file listing, per read in the BAM file,\nthe value\
\ for the \"CB\", \"UX\", \"GX\" and \"GN\" tag, together with the\nchromsome\
\ to which the read was mapped to.\n"
info: null
default:
- "processedBamFile.txt"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--nrReadsNrGenesPerChrom"
description: "Path to an output file that contains a .tsv formatted table describing\n\
per chromosome the number of reads that were mapped to that chromosome (NumberOfReads\n\
column) and the number of genes on that chromosome that had at least one\nread\
\ mapped to it (NumberOfGenes).\n"
info: null
default:
- "nrReadsNrGenesPerChrom.txt"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--nrReadsNrUMIsPerCB"
description: "Path to an output file that contains a .tsv formatted table describing\n\
per barcode the number of UMI's (nrUMIs) and the total number of reads (NumberOfReads).\n"
info: null
default:
- "nrReadsNrUMIsPerCB.txt"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--umiFreqTop"
description: "Path to an output file that contains a .tsv formatted table describing\n\
per UMI (column UB) the frequency at which they occur in the reads (column\n\
N). Only the top 100 UMIs are included.\n"
info: null
default:
- "umiFreqTop100.txt"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "integer"
name: "--threads"
description: "Number of threads to use for decompressing BAM files.\n"
info: null
default:
- 1
required: false
min: 1
direction: "input"
multiple: false
multiple_sep: ";"
resources:
- type: "python_script"
path: "script.py"
is_executable: true
- type: "file"
path: "nextflow_labels.config"
dest: "nextflow_labels.config"
description: "Generate summary statistics from BAM files generated by STAR solo."
test_resources:
- type: "python_script"
path: "test.py"
is_executable: true
- type: "file"
path: "test.sam"
info: null
status: "enabled"
requirements:
commands:
- "ps"
license: "MIT"
links:
repository: "https://github.com/viash-hub/htrnaseq"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
script:
- "includeConfig(\"nextflow_labels.config\")"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "debian:stable-slim"
target_registry: "images.viash-hub.com"
target_tag: "main"
namespace_separator: "/"
setup:
- type: "docker"
env:
- "PIP_BREAK_SYSTEM_PACKAGES=1"
- "HTSLIB_LIBRARY_DIR=/usr/lib/"
- "HTSLIB_INCLUDE_DIR=/usr/include/"
- type: "apt"
packages:
- "python3"
- "python3-pip"
- "python3-venv"
- "python-is-python3"
- "libhts-dev"
- "procps"
interactive: false
- type: "python"
user: false
packages:
- "pysam"
- "pandas"
upgrade: true
test_setup:
- type: "python"
user: false
packages:
- "viashpy"
upgrade: true
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/stats/generate_well_statistics/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/stats/generate_well_statistics"
executable: "target/executable/stats/generate_well_statistics/generate_well_statistics"
viash_version: "0.9.0-RC7"
git_commit: "b98f6367d672368af134843711a46d3b53717187"
git_remote: "https://github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\
\ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\
\ dest: 'nextflow_labels.config'}\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"
keywords:
- "bioinformatics"
- "sequence"
- "high-throughput"
- "mapping"
- "counting"
- "pipeline"
license: "MIT"
organization: "vsh"
links:
repository: "https://github.com/viash-hub/htrnaseq"
issue_tracker: "https://github.com/viash-hub/htrnaseq/issues"

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,43 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare
}
try {
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
return process.maxMemory
}
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
return max_memory as nextflow.util.MemoryUnit
}
else {
return to_compare
}
} catch (all) {
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
System.exit(1)
}
}