Build branch main with version main (8eb391e)
Build pipeline: viash-hub.htrnaseq.main-xhqqt
Source commit: 8eb391e3eb
Source message: Revert Viash version back to v0.9.0
This commit is contained in:
@@ -111,7 +111,7 @@ argument_groups:
|
||||
description: "Output from mapping with STAR"
|
||||
info: null
|
||||
default:
|
||||
- "$id/star/*"
|
||||
- "star.$id/*"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
@@ -177,7 +177,7 @@ argument_groups:
|
||||
name: "--html_report"
|
||||
info: null
|
||||
default:
|
||||
- "report.html"
|
||||
- "report.$id.html"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
@@ -319,8 +319,8 @@ build_info:
|
||||
output: "target/nextflow/workflows/htrnaseq"
|
||||
executable: "target/nextflow/workflows/htrnaseq/main.nf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "d8c0c0f518e9359c5373ff2761a80050ebb7333e"
|
||||
git_remote: "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
git_commit: "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb"
|
||||
git_remote: "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
dependencies:
|
||||
- "target/nextflow/stats/combine_star_logs"
|
||||
- "target/nextflow/stats/generate_pool_statistics"
|
||||
|
||||
@@ -2940,7 +2940,7 @@ meta = [
|
||||
"name" : "--star_output",
|
||||
"description" : "Output from mapping with STAR",
|
||||
"default" : [
|
||||
"$id/star/*"
|
||||
"star.$id/*"
|
||||
],
|
||||
"must_exist" : true,
|
||||
"create_parent" : true,
|
||||
@@ -3018,7 +3018,7 @@ meta = [
|
||||
"type" : "file",
|
||||
"name" : "--html_report",
|
||||
"default" : [
|
||||
"report.html"
|
||||
"report.$id.html"
|
||||
],
|
||||
"must_exist" : true,
|
||||
"create_parent" : true,
|
||||
@@ -3223,8 +3223,8 @@ meta = [
|
||||
"engine" : "native|native",
|
||||
"output" : "target/nextflow/workflows/htrnaseq",
|
||||
"viash_version" : "0.9.0",
|
||||
"git_commit" : "d8c0c0f518e9359c5373ff2761a80050ebb7333e",
|
||||
"git_remote" : "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
"git_commit" : "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb",
|
||||
"git_remote" : "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
},
|
||||
"package_config" : {
|
||||
"name" : "htrnaseq",
|
||||
@@ -3290,7 +3290,10 @@ workflow run_wf {
|
||||
f_data_ch = input_ch
|
||||
| create_fdata.run(
|
||||
directives: [label: ["lowmem", "lowcpu"]],
|
||||
fromState: ["gtf": "annotation"],
|
||||
fromState: [
|
||||
"gtf": "annotation",
|
||||
"output": "f_data"
|
||||
],
|
||||
toState: {id, result, state -> ["f_data": result.output]}
|
||||
)
|
||||
|
||||
@@ -3341,7 +3344,7 @@ workflow run_wf {
|
||||
]
|
||||
},
|
||||
toState: [
|
||||
"nrReadsNrGenesPerChrom": "nrReadsNrGenesPerChrom",
|
||||
"nrReadsNrGenesPerChromWell": "nrReadsNrGenesPerChrom",
|
||||
]
|
||||
)
|
||||
| map {id, state ->
|
||||
@@ -3373,7 +3376,10 @@ workflow run_wf {
|
||||
"barcode": barcodes,
|
||||
"well_id": well_ids,
|
||||
"star_output": states.collect{it.star_output},
|
||||
"nrReadsNrGenesPerChrom": states.collect{it.nrReadsNrGenesPerChrom},
|
||||
// Well and pool stats should be carefully kept separate.
|
||||
// The workflow argument points to the name for the pool statistics:
|
||||
"nrReadsNrGenesPerChromWell": states.collect{it.nrReadsNrGenesPerChromWell},
|
||||
"nrReadsNrGenesPerChromPool": states[0].nrReadsNrGenesPerChrom
|
||||
]
|
||||
//For many state items, the value is the same across states.
|
||||
def other_state_keys = states.inject([].toSet()){ current_keys, state ->
|
||||
@@ -3401,7 +3407,8 @@ workflow run_wf {
|
||||
| generate_pool_statistics.run(
|
||||
directives: ["label": ["lowmem", "verylowcpu"]],
|
||||
fromState: [
|
||||
"nrReadsNrGenesPerChrom": "nrReadsNrGenesPerChrom",
|
||||
"nrReadsNrGenesPerChrom": "nrReadsNrGenesPerChromWell",
|
||||
"nrReadsNrGenesPerChromPool": "nrReadsNrGenesPerChromPool"
|
||||
],
|
||||
toState: [
|
||||
"nrReadsNrGenesPerChromPool": "nrReadsNrGenesPerChromPool"
|
||||
@@ -3436,6 +3443,7 @@ workflow run_wf {
|
||||
fromState: [
|
||||
"star_stats_file": "star_qc_metrics",
|
||||
"nrReadsNrGenesPerChromPool": "nrReadsNrGenesPerChromPool",
|
||||
"output": "p_data"
|
||||
],
|
||||
toState: ["p_data": "output"],
|
||||
)
|
||||
|
||||
@@ -110,10 +110,10 @@
|
||||
"star_output": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: List of `file`, required, default: `$id.$key.star_output_*.star_output_*`, multiple_sep: `\";\"`. Output from mapping with STAR",
|
||||
"help_text": "Type: List of `file`, required, default: `$id.$key.star_output_*.star_output_*`, multiple_sep: `\";\"`. Output from mapping with STAR"
|
||||
"description": "Type: List of `file`, required, default: `$id.$key.star_output_*.$id/*`, multiple_sep: `\";\"`. Output from mapping with STAR",
|
||||
"help_text": "Type: List of `file`, required, default: `$id.$key.star_output_*.$id/*`, multiple_sep: `\";\"`. Output from mapping with STAR"
|
||||
,
|
||||
"default":"$id.$key.star_output_*.star_output_*"
|
||||
"default":"$id.$key.star_output_*.$id/*"
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -183,8 +183,8 @@ build_info:
|
||||
output: "target/nextflow/workflows/parallel_map_wf"
|
||||
executable: "target/nextflow/workflows/parallel_map_wf/main.nf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "d8c0c0f518e9359c5373ff2761a80050ebb7333e"
|
||||
git_remote: "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
git_commit: "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb"
|
||||
git_remote: "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
dependencies:
|
||||
- "target/nextflow/parallel_map"
|
||||
- "target/nextflow/workflows/utils/groupWells"
|
||||
|
||||
@@ -3037,8 +3037,8 @@ meta = [
|
||||
"engine" : "native|native",
|
||||
"output" : "target/nextflow/workflows/parallel_map_wf",
|
||||
"viash_version" : "0.9.0",
|
||||
"git_commit" : "d8c0c0f518e9359c5373ff2761a80050ebb7333e",
|
||||
"git_remote" : "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
"git_commit" : "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb",
|
||||
"git_remote" : "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
},
|
||||
"package_config" : {
|
||||
"name" : "htrnaseq",
|
||||
|
||||
236
target/nextflow/workflows/runner/.config.vsh.yaml
Normal file
236
target/nextflow/workflows/runner/.config.vsh.yaml
Normal file
@@ -0,0 +1,236 @@
|
||||
name: "runner"
|
||||
namespace: "workflows"
|
||||
version: "main"
|
||||
argument_groups:
|
||||
- name: "Input arguments"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--input"
|
||||
description: "Base directory of the form `s3:/<bucket>/Sequencing/<Sequencer>/<RunID>/<demultiplex_dir>`"
|
||||
info: null
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--barcodesFasta"
|
||||
info: null
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--genomeDir"
|
||||
info: null
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--annotation"
|
||||
info: null
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- name: "Metadata arguments"
|
||||
arguments:
|
||||
- type: "string"
|
||||
name: "--id"
|
||||
description: "Unique identifier for the run"
|
||||
info: null
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--project_id"
|
||||
description: "Project ID"
|
||||
info: null
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--experiment_id"
|
||||
description: "Experiment ID"
|
||||
info: null
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- name: "Annotation flags"
|
||||
arguments:
|
||||
- type: "boolean_true"
|
||||
name: "--plain_output"
|
||||
description: "Flag to indicate that the output should be stored directly under\
|
||||
\ $publish_dir rather than\nunder a subdirectory structure runID/<date_time>_demultiplex_<version>/.\n"
|
||||
info: null
|
||||
direction: "input"
|
||||
- name: "Publish arguments"
|
||||
arguments:
|
||||
- type: "string"
|
||||
name: "--fastq_publish_dir"
|
||||
info: null
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--results_publish_dir"
|
||||
info: null
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "nextflow_script"
|
||||
path: "main.nf"
|
||||
is_executable: true
|
||||
entrypoint: "run_wf"
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Runner for HT RNA-seq pipeline"
|
||||
info: null
|
||||
status: "enabled"
|
||||
requirements:
|
||||
commands:
|
||||
- "ps"
|
||||
dependencies:
|
||||
- name: "utils/listInputDir"
|
||||
repository:
|
||||
type: "local"
|
||||
- name: "workflows/htrnaseq"
|
||||
repository:
|
||||
type: "local"
|
||||
- name: "io/publish_fastqs"
|
||||
repository:
|
||||
type: "local"
|
||||
- name: "io/publish_results"
|
||||
repository:
|
||||
type: "local"
|
||||
license: "MIT"
|
||||
links:
|
||||
repository: "https://github.com/viash-hub/htrnaseq"
|
||||
runners:
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "native"
|
||||
id: "native"
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/workflows/runner/config.vsh.yaml"
|
||||
runner: "nextflow"
|
||||
engine: "native|native"
|
||||
output: "target/nextflow/workflows/runner"
|
||||
executable: "target/nextflow/workflows/runner/main.nf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb"
|
||||
git_remote: "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
dependencies:
|
||||
- "target/nextflow/utils/listInputDir"
|
||||
- "target/nextflow/workflows/htrnaseq"
|
||||
- "target/nextflow/io/publish_fastqs"
|
||||
- "target/nextflow/io/publish_results"
|
||||
package_config:
|
||||
name: "htrnaseq"
|
||||
version: "main"
|
||||
description: "High-throughput pipeline [WIP]\n"
|
||||
info:
|
||||
test_resources:
|
||||
- path: "gs://viash-hub-test-data/htrnaseq/v1/"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\
|
||||
\ dest: 'nextflow_labels.config'}\n"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'main'"
|
||||
keywords:
|
||||
- "bioinformatics"
|
||||
- "sequence"
|
||||
- "high-throughput"
|
||||
- "mapping"
|
||||
- "counting"
|
||||
- "pipeline"
|
||||
license: "MIT"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/viash-hub/htrnaseq"
|
||||
issue_tracker: "https://github.com/viash-hub/htrnaseq/issues"
|
||||
3431
target/nextflow/workflows/runner/main.nf
Normal file
3431
target/nextflow/workflows/runner/main.nf
Normal file
File diff suppressed because it is too large
Load Diff
125
target/nextflow/workflows/runner/nextflow.config
Normal file
125
target/nextflow/workflows/runner/nextflow.config
Normal file
@@ -0,0 +1,125 @@
|
||||
manifest {
|
||||
name = 'workflows/runner'
|
||||
mainScript = 'main.nf'
|
||||
nextflowVersion = '!>=20.12.1-edge'
|
||||
version = 'main'
|
||||
description = 'Runner for HT RNA-seq pipeline'
|
||||
}
|
||||
|
||||
process.container = 'nextflow/bash:latest'
|
||||
|
||||
// detect tempdir
|
||||
tempDir = java.nio.file.Paths.get(
|
||||
System.getenv('NXF_TEMP') ?:
|
||||
System.getenv('VIASH_TEMP') ?:
|
||||
System.getenv('TEMPDIR') ?:
|
||||
System.getenv('TMPDIR') ?:
|
||||
'/tmp'
|
||||
).toAbsolutePath()
|
||||
|
||||
profiles {
|
||||
no_publish {
|
||||
process {
|
||||
withName: '.*' {
|
||||
publishDir = [
|
||||
enabled: false
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
mount_temp {
|
||||
docker.temp = tempDir
|
||||
podman.temp = tempDir
|
||||
charliecloud.temp = tempDir
|
||||
}
|
||||
docker {
|
||||
docker.enabled = true
|
||||
// docker.userEmulation = true
|
||||
singularity.enabled = false
|
||||
podman.enabled = false
|
||||
shifter.enabled = false
|
||||
charliecloud.enabled = false
|
||||
}
|
||||
singularity {
|
||||
singularity.enabled = true
|
||||
singularity.autoMounts = true
|
||||
docker.enabled = false
|
||||
podman.enabled = false
|
||||
shifter.enabled = false
|
||||
charliecloud.enabled = false
|
||||
}
|
||||
podman {
|
||||
podman.enabled = true
|
||||
docker.enabled = false
|
||||
singularity.enabled = false
|
||||
shifter.enabled = false
|
||||
charliecloud.enabled = false
|
||||
}
|
||||
shifter {
|
||||
shifter.enabled = true
|
||||
docker.enabled = false
|
||||
singularity.enabled = false
|
||||
podman.enabled = false
|
||||
charliecloud.enabled = false
|
||||
}
|
||||
charliecloud {
|
||||
charliecloud.enabled = true
|
||||
docker.enabled = false
|
||||
singularity.enabled = false
|
||||
podman.enabled = false
|
||||
shifter.enabled = false
|
||||
}
|
||||
}
|
||||
|
||||
process{
|
||||
withLabel: mem1gb { memory = 1000000000.B }
|
||||
withLabel: mem2gb { memory = 2000000000.B }
|
||||
withLabel: mem5gb { memory = 5000000000.B }
|
||||
withLabel: mem10gb { memory = 10000000000.B }
|
||||
withLabel: mem20gb { memory = 20000000000.B }
|
||||
withLabel: mem50gb { memory = 50000000000.B }
|
||||
withLabel: mem100gb { memory = 100000000000.B }
|
||||
withLabel: mem200gb { memory = 200000000000.B }
|
||||
withLabel: mem500gb { memory = 500000000000.B }
|
||||
withLabel: mem1tb { memory = 1000000000000.B }
|
||||
withLabel: mem2tb { memory = 2000000000000.B }
|
||||
withLabel: mem5tb { memory = 5000000000000.B }
|
||||
withLabel: mem10tb { memory = 10000000000000.B }
|
||||
withLabel: mem20tb { memory = 20000000000000.B }
|
||||
withLabel: mem50tb { memory = 50000000000000.B }
|
||||
withLabel: mem100tb { memory = 100000000000000.B }
|
||||
withLabel: mem200tb { memory = 200000000000000.B }
|
||||
withLabel: mem500tb { memory = 500000000000000.B }
|
||||
withLabel: mem1gib { memory = 1073741824.B }
|
||||
withLabel: mem2gib { memory = 2147483648.B }
|
||||
withLabel: mem4gib { memory = 4294967296.B }
|
||||
withLabel: mem8gib { memory = 8589934592.B }
|
||||
withLabel: mem16gib { memory = 17179869184.B }
|
||||
withLabel: mem32gib { memory = 34359738368.B }
|
||||
withLabel: mem64gib { memory = 68719476736.B }
|
||||
withLabel: mem128gib { memory = 137438953472.B }
|
||||
withLabel: mem256gib { memory = 274877906944.B }
|
||||
withLabel: mem512gib { memory = 549755813888.B }
|
||||
withLabel: mem1tib { memory = 1099511627776.B }
|
||||
withLabel: mem2tib { memory = 2199023255552.B }
|
||||
withLabel: mem4tib { memory = 4398046511104.B }
|
||||
withLabel: mem8tib { memory = 8796093022208.B }
|
||||
withLabel: mem16tib { memory = 17592186044416.B }
|
||||
withLabel: mem32tib { memory = 35184372088832.B }
|
||||
withLabel: mem64tib { memory = 70368744177664.B }
|
||||
withLabel: mem128tib { memory = 140737488355328.B }
|
||||
withLabel: mem256tib { memory = 281474976710656.B }
|
||||
withLabel: mem512tib { memory = 562949953421312.B }
|
||||
withLabel: cpu1 { cpus = 1 }
|
||||
withLabel: cpu2 { cpus = 2 }
|
||||
withLabel: cpu5 { cpus = 5 }
|
||||
withLabel: cpu10 { cpus = 10 }
|
||||
withLabel: cpu20 { cpus = 20 }
|
||||
withLabel: cpu50 { cpus = 50 }
|
||||
withLabel: cpu100 { cpus = 100 }
|
||||
withLabel: cpu200 { cpus = 200 }
|
||||
withLabel: cpu500 { cpus = 500 }
|
||||
withLabel: cpu1000 { cpus = 1000 }
|
||||
}
|
||||
|
||||
includeConfig("nextflow_labels.config")
|
||||
108
target/nextflow/workflows/runner/nextflow_labels.config
Normal file
108
target/nextflow/workflows/runner/nextflow_labels.config
Normal file
@@ -0,0 +1,108 @@
|
||||
executor {
|
||||
$k8s {
|
||||
submitRateLimit = '10sec'
|
||||
pollInterval = '1 sec'
|
||||
}
|
||||
}
|
||||
|
||||
process {
|
||||
container = 'nextflow/bash:latest'
|
||||
|
||||
// default resources
|
||||
memory = { 8.Gb * task.attempt }
|
||||
cpus = 8
|
||||
maxForks = 36
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = 192.GB
|
||||
|
||||
// Resource labels
|
||||
withLabel: verylowcpu { cpus = 2 }
|
||||
withLabel: lowcpu { cpus = 8 }
|
||||
withLabel: midcpu { cpus = 16 }
|
||||
withLabel: highcpu { cpus = 32 }
|
||||
|
||||
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
|
||||
|
||||
}
|
||||
|
||||
profiles {
|
||||
// detect tempdir
|
||||
tempDir = java.nio.file.Paths.get(
|
||||
System.getenv('NXF_TEMP') ?:
|
||||
System.getenv('VIASH_TEMP') ?:
|
||||
System.getenv('TEMPDIR') ?:
|
||||
System.getenv('TMPDIR') ?:
|
||||
'/tmp'
|
||||
).toAbsolutePath()
|
||||
|
||||
mount_temp {
|
||||
docker.temp = tempDir
|
||||
podman.temp = tempDir
|
||||
charliecloud.temp = tempDir
|
||||
}
|
||||
|
||||
no_publish {
|
||||
process {
|
||||
withName: '.*' {
|
||||
publishDir = [
|
||||
enabled: false
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
docker {
|
||||
docker.fixOwnership = true
|
||||
docker.enabled = true
|
||||
// docker.userEmulation = true
|
||||
singularity.enabled = false
|
||||
podman.enabled = false
|
||||
shifter.enabled = false
|
||||
charliecloud.enabled = false
|
||||
}
|
||||
|
||||
local {
|
||||
// This config is for local processing.
|
||||
process {
|
||||
withName: ".*parallel_map_process" {
|
||||
maxForks = 1
|
||||
}
|
||||
maxMemory = 25.GB
|
||||
withLabel: verylowcpu { cpus = 2 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 6 }
|
||||
withLabel: highcpu { cpus = 8 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
203
target/nextflow/workflows/runner/nextflow_schema.json
Normal file
203
target/nextflow/workflows/runner/nextflow_schema.json
Normal file
@@ -0,0 +1,203 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema",
|
||||
"title": "runner",
|
||||
"description": "Runner for HT RNA-seq pipeline",
|
||||
"type": "object",
|
||||
"definitions": {
|
||||
|
||||
|
||||
|
||||
"input arguments" : {
|
||||
"title": "Input arguments",
|
||||
"type": "object",
|
||||
"description": "No description",
|
||||
"properties": {
|
||||
|
||||
|
||||
"input": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/\u003cdemultiplex_dir\u003e`",
|
||||
"help_text": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/\u003cdemultiplex_dir\u003e`"
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"barcodesFasta": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `file`, required. ",
|
||||
"help_text": "Type: `file`, required. "
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"genomeDir": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `file`, required. ",
|
||||
"help_text": "Type: `file`, required. "
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"annotation": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `file`, required. ",
|
||||
"help_text": "Type: `file`, required. "
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
"metadata arguments" : {
|
||||
"title": "Metadata arguments",
|
||||
"type": "object",
|
||||
"description": "No description",
|
||||
"properties": {
|
||||
|
||||
|
||||
"id": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`. Unique identifier for the run",
|
||||
"help_text": "Type: `string`. Unique identifier for the run"
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"project_id": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, required. Project ID",
|
||||
"help_text": "Type: `string`, required. Project ID"
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"experiment_id": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, required. Experiment ID",
|
||||
"help_text": "Type: `string`, required. Experiment ID"
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
"annotation flags" : {
|
||||
"title": "Annotation flags",
|
||||
"type": "object",
|
||||
"description": "No description",
|
||||
"properties": {
|
||||
|
||||
|
||||
"plain_output": {
|
||||
"type":
|
||||
"boolean",
|
||||
"description": "Type: `boolean_true`, default: `false`. Flag to indicate that the output should be stored directly under $publish_dir rather than\nunder a subdirectory structure runID/\u003cdate_time\u003e_demultiplex_\u003cversion\u003e/",
|
||||
"help_text": "Type: `boolean_true`, default: `false`. Flag to indicate that the output should be stored directly under $publish_dir rather than\nunder a subdirectory structure runID/\u003cdate_time\u003e_demultiplex_\u003cversion\u003e/.\n"
|
||||
,
|
||||
"default":false
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
"publish arguments" : {
|
||||
"title": "Publish arguments",
|
||||
"type": "object",
|
||||
"description": "No description",
|
||||
"properties": {
|
||||
|
||||
|
||||
"fastq_publish_dir": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, required. ",
|
||||
"help_text": "Type: `string`, required. "
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"results_publish_dir": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, required. ",
|
||||
"help_text": "Type: `string`, required. "
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
},
|
||||
|
||||
|
||||
"nextflow input-output arguments" : {
|
||||
"title": "Nextflow input-output arguments",
|
||||
"type": "object",
|
||||
"description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
|
||||
"properties": {
|
||||
|
||||
|
||||
"publish_dir": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, required, example: `output/`. Path to an output directory",
|
||||
"help_text": "Type: `string`, required, example: `output/`. Path to an output directory."
|
||||
|
||||
}
|
||||
|
||||
|
||||
,
|
||||
"param_list": {
|
||||
"type":
|
||||
"string",
|
||||
"description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel",
|
||||
"help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.",
|
||||
"hidden": true
|
||||
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
},
|
||||
"allOf": [
|
||||
|
||||
{
|
||||
"$ref": "#/definitions/input arguments"
|
||||
},
|
||||
|
||||
{
|
||||
"$ref": "#/definitions/metadata arguments"
|
||||
},
|
||||
|
||||
{
|
||||
"$ref": "#/definitions/annotation flags"
|
||||
},
|
||||
|
||||
{
|
||||
"$ref": "#/definitions/publish arguments"
|
||||
},
|
||||
|
||||
{
|
||||
"$ref": "#/definitions/nextflow input-output arguments"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -185,8 +185,8 @@ build_info:
|
||||
output: "target/nextflow/workflows/utils/groupWells"
|
||||
executable: "target/nextflow/workflows/utils/groupWells/main.nf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "d8c0c0f518e9359c5373ff2761a80050ebb7333e"
|
||||
git_remote: "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
git_commit: "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb"
|
||||
git_remote: "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
package_config:
|
||||
name: "htrnaseq"
|
||||
version: "main"
|
||||
|
||||
@@ -3039,8 +3039,8 @@ meta = [
|
||||
"engine" : "native",
|
||||
"output" : "target/nextflow/workflows/utils/groupWells",
|
||||
"viash_version" : "0.9.0",
|
||||
"git_commit" : "d8c0c0f518e9359c5373ff2761a80050ebb7333e",
|
||||
"git_remote" : "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
"git_commit" : "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb",
|
||||
"git_remote" : "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
},
|
||||
"package_config" : {
|
||||
"name" : "htrnaseq",
|
||||
|
||||
@@ -255,8 +255,8 @@ build_info:
|
||||
output: "target/nextflow/workflows/well_demultiplex"
|
||||
executable: "target/nextflow/workflows/well_demultiplex/main.nf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "d8c0c0f518e9359c5373ff2761a80050ebb7333e"
|
||||
git_remote: "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
git_commit: "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb"
|
||||
git_remote: "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
dependencies:
|
||||
- "target/dependencies/vsh/vsh/biobox/v0.3.0/nextflow/cutadapt"
|
||||
- "target/dependencies/vsh/vsh/craftbox/v0.1.0/nextflow/concat_text"
|
||||
|
||||
@@ -3134,8 +3134,8 @@ meta = [
|
||||
"engine" : "native|native",
|
||||
"output" : "target/nextflow/workflows/well_demultiplex",
|
||||
"viash_version" : "0.9.0",
|
||||
"git_commit" : "d8c0c0f518e9359c5373ff2761a80050ebb7333e",
|
||||
"git_remote" : "https://x-access-token:ghs_WwvhrfwIl9DJdsjBAWPXvIvgBFDL7344Rs7f@github.com/viash-hub/htrnaseq"
|
||||
"git_commit" : "8eb391e3ebd9ffe573b76b9be635a81ec495c3bb",
|
||||
"git_remote" : "https://x-access-token:ghs_ybw8wXfG5bR3FsTslxlh1sXKguvM9R0Rd31p@github.com/viash-hub/htrnaseq"
|
||||
},
|
||||
"package_config" : {
|
||||
"name" : "htrnaseq",
|
||||
@@ -3262,7 +3262,8 @@ workflow run_wf {
|
||||
| cutadapt.run(
|
||||
directives: [label: ["highmem", "midcpu"]],
|
||||
fromState: { id, state ->
|
||||
def new_output = ("fastq_${state.lane_sorting}/*_001.fastq")
|
||||
// Remark: the fastq path part may seem superfluous but is necessary for publising later
|
||||
def new_output = ("fastq/${id}/*_001.fastq")
|
||||
[
|
||||
input: state.input_r1,
|
||||
input_r2: state.input_r2,
|
||||
|
||||
Reference in New Issue
Block a user