vsh/demultiplex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch main with version main (64a371e)

Browse Source 
Build pipeline: viash-hub.demultiplex.main-lcwkk

Source commit: 64a371e168

Source message: Update CHANGELOG for 0.2.0 (#25)
This commit is contained in:
CI
2024-12-11 09:03:18 +00:00
parent fcc3a4d62e
commit 73c077fd82
36 changed files with 9615 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
CHANGELOG.md
View File

@@ -4,6 +4,36 @@
* `demultiplex` workflow: renamed `sample_sheet` argument to `run_information` (PR #24)
## Major updates
The outflow of the workflow has been refactored to be more flexible (PR #19). This is done by creating a wrapper workflow `runner` that wraps the native `demultiplex` workflow. The `runner` workflow is responsible for setting the output directory based on the input arguments:
3 arguments exist for specifying the relative location of the 3 _outputs_ of the workflow:
- `fastq_output`: The directory where the demultiplexed fastq files are stored.
- `falco_output`: the directory for the `fastqc`/`falco` reports.
- `multiqc_output`: The filename for the `multiqc` report.
The target location path is determined by the following logic:
- If no `id` is provided, the output directory is set to `$publish_dir`.
- If an `id` is explicitly set using Seqera Cloud or by adding `--id <>`, the output directory is set to `$publish_dir/<id>`.
The workflow has two optional flags to be used in combination with `--id`:
- `--add_date_time`: rather than publishing the results under `$publish_dir`, this adds an additional layer `$publish_dir/<date-time-stamp>/`. This is useful when you want to keep track of multiple runs of the workflow (example: `240322_143020`).
- `--add_workflow_id`: adding this flag will add `_demultiplex_<version>` to the output directory (example: `demultiplex_v0.2.0`). When starting the workflow from a non-release, the version will be set to `version_unkonwn`.
The default structure in the output directory is:
- Two sub-directories:
- `fastq`
- `qc` for the reports:
- `multiqc_report.html`
- `fastqc/` directory containing the different fastqc (falco) reports.
The `$publish_dir` variable corresponds to the argument provided with `--publish-dir`. The `date-time-stamp` is generated by the workflow based on when it was launched and is thus guaranteed to be unique.
## New features
* Add support for `bases2fastq` demultiplexer (PR #24)

1
_viash.yaml
View File

@@ -15,3 +15,4 @@ viash_version: 0.9.0
config_mods: |
.requirements.commands := ['ps']
.runners[.type == 'nextflow'].directives.tag := '$id'

14
nextflow.config
View File

@@ -1,6 +1,12 @@
manifest {
name = "demultiplex"
version = "main"
defaultBranch = "main"
nextflowVersion = "!>=20.12.1-edge"
homePage = 'https://github.com/viash-hub/demultiplex'
description = 'Demultiplexing pipeline for sequencing data'
mainScript = 'target/nextflow/demultiplex/main.nf'
}
process {
withName: publishStatesProc {
publishDir = [ enabled: false ]
}
}

10
src/demultiplex/config.vsh.yaml
View File

@@ -3,6 +3,9 @@ description: Demultiplexing of raw sequencing data
argument_groups:
- name: Input arguments
arguments:
- name: --id
description: Unique identifier for the run
type: string
- name: --input
description: Directory containing raw sequencing data
type: file
@@ -31,19 +34,20 @@ argument_groups:
description: Directory to write fastq data to
type: file
direction: output
required: true
required: false
default: "$id/fastq"
- name: "--output_falco"
description: Directory to write falco output to
type: file
direction: output
required: false
default: "$id/falco"
default: "$id/qc/fastqc"
- name: "--output_multiqc"
description: Directory to write falco output to
type: file
direction: output
required: false
default: "$id/multiqc_report.html"
default: "$id/qc/multiqc_report.html"
resources:
- type: nextflow_script
path: main.nf

2
src/demultiplex/integration_tests.sh
View File

@@ -6,7 +6,7 @@ REPO_ROOT=$(git rev-parse --show-toplevel)
# ensure that the command below is run from the root of the repository
cd "$REPO_ROOT"
viash ns build --setup cb
viash ns build --setup cb -q demultiplex
nextflow run . \
-main-script src/demultiplex/test.nf \

24
src/demultiplex/main.nf
View File

@@ -4,6 +4,7 @@ workflow run_wf {
main:
samples_ch = input_ch
// untar input if needed
| untar.run(
directives: [label: ["lowmem", "lowcpu"]],
@@ -109,11 +110,15 @@ workflow run_wf {
| bcl_convert.run(
runIf: {id, state -> state.demultiplexer in ["bclconvert"]},
directives: [label: ["highmem", "midcpu"]],
fromState: [
"bcl_input_directory": "input",
"sample_sheet": "run_information",
"output_directory": "output",
],
fromState: { id, state ->
[
bcl_input_directory: state.input,
sample_sheet: state.run_information,
output_directory: state.output,
reports: "reports",
logs: "logs"
]
},
toState: {id, result, state ->
def toAdd = [
"output_demultiplexer" : result.output_directory,
@@ -159,6 +164,8 @@ workflow run_wf {
)
output_ch = samples_ch
| combine_samples.run(
fromState: { id, state ->
[
@@ -186,10 +193,10 @@ workflow run_wf {
},
toState: { id, result, state ->
state + [ "output_falco" : result.outdir ]
},
}
)
| multiqc.run(
directives: [label: ["lowcpu", "lowmem"]],
directives: [label: ["midcpu", "midmem"]],
fromState: {id, state ->
def new_state = [
"input": [state.output_falco],
@@ -206,10 +213,11 @@ workflow run_wf {
},
toState: { id, result, state ->
state + [ "output_multiqc" : result.output_report ]
},
}
)
| setState(
[
//"_meta": "_meta",
"output": "output_demultiplexer",
"output_falco": "output_falco",
"output_multiqc": "output_multiqc"

30
src/io/publish/code.sh Executable file
View File

@@ -0,0 +1,30 @@
#!/bin/bash
echo "Publishing $par_input -> $par_output"
echo "Publishing $par_input_falco -> $par_output_falco"
echo "Publishing $par_input_multiqc -> $par_output_multiqc"
echo
echo "Creating directory if it does not exist:"
mkdir -p $(dirname "$par_output") && echo "Containing directory $par_output created"
mkdir -p $(dirname "$par_output_falco") && echo "Containing directory $par_output_falco created"
mkdir -p $(dirname "$par_output_multiqc") && echo "Containing directory $par_output_multiqc created"
echo
echo "Copying files..."
cp -rL "$par_input" "$par_output"
cp -rL "$par_input_falco" "$par_output_falco"
cp -rL "$par_input_multiqc" "$par_output_multiqc"
echo
echo "Output files:"
echo "par_output:"
ls "$par_output"
echo
echo "par_output_falco:"
ls "$par_output_falco"
echo
echo "par_output_multiqc:"
ls "$par_output_multiqc"

48
src/io/publish/config.vsh.yaml Normal file
View File

@@ -0,0 +1,48 @@
name: "publish"
namespace: "io"
description: "Publish the processed results of the run"
argument_groups:
- name: Input arguments
arguments:
- name: --input
description: Directory to write fastq data to
type: file
required: true
- name: "--input_falco"
description: Directory to write falco output to
type: file
required: true
- name: "--input_multiqc"
description: Directory to write falco output to
type: file
required: true
- name: Output arguments
arguments:
- name: --output
type: file
direction: output
default: "fastq"
- name: --output_falco
type: file
direction: output
default: "qc/fastqc"
- name: --output_multiqc
type: file
direction: output
default: "qc/multiqc_report.html"
resources:
- type: bash_script
path: ./code.sh
engines:
- type: docker
image: debian:stable-slim
setup:
- type: apt
packages:
- procps
runners:
- type: executable
- type: nextflow

52
src/runner/config.vsh.yaml Normal file
View File

@@ -0,0 +1,52 @@
name: runner
description: Runner for demultiplexing of raw sequencing data
argument_groups:
- name: Input arguments
arguments:
- name: --input
description: Base directory of the form `s3:/<bucket>/Sequencing/<Sequencer>/<RunID>/`
type: file
required: true
- name: Annotation flags
arguments:
- name: --add_date_time
description: |
Add date and time to the output directory name. This is useful
when running the same pipeline multiple times on the same input
directory.
type: boolean_true
- name: --add_workflow_id
description: |
Add a workflow identifier to the output directory name.
type: boolean_true
- name: Output arguments
arguments:
- name: --fastq_output
type: file
direction: output
default: "fastq"
- name: --falco_output
type: file
direction: output
default: "qc/fastqc"
- name: --multiqc_output
type: file
direction: output
default: "qc/multiqc_report.html"
resources:
- type: nextflow_script
path: main.nf
entrypoint: run_wf
dependencies:
- name: demultiplex
repository: local
- name: io/publish
repository: local
runners:
- type: nextflow
engines:
- type: native

67
src/runner/main.nf Normal file
View File

@@ -0,0 +1,67 @@
def date = new Date().format('yyyyMMdd_hhmmss')
def viash_config = java.nio.file.Paths.get("$projectDir/../../../").toAbsolutePath().normalize().toString() + "/_viash.yaml"
def version = get_version(viash_config)
workflow run_wf {
take:
input_ch
main:
output_ch = input_ch
| demultiplex.run(
fromState: [
"input": "input",
"output": "fastq",
"output_falco": "qc/fastqc",
"output_multiqc": "qc/multiqc_report.html",
],
toState: { id, result, state ->
state + result
},
)
| publish.run(
fromState: { id, state ->
def id1 = (params.add_date_time) ? "${id}_${date}" : id
def id2 = (params.add_workflow_id) ? "${id1}_demultiplex_${version}" : id1
def fastq_output_1 = (id == "run") ? state.fastq_output : "${id2}/" + state.fastq_output
def falco_output_1 = (id == "run") ? state.falco_output : "${id2}/" + state.falco_output
def multiqc_output_1 = (id == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output
if (id == "run") {
println("Publising to ${params.publish_dir}")
} else {
println("Publising to ${params.publish_dir}/${id2}")
}
[
input: state.output,
input_falco: state.output_falco,
input_multiqc: state.output_multiqc,
output: fastq_output_1,
output_falco: falco_output_1,
output_multiqc: multiqc_output_1
]
},
toState: { id, result, state -> [:] },
directives: [
publishDir: [
path: "${params.publish_dir}",
overwrite: false,
mode: "copy"
]
]
)
emit:
output_ch
}
def get_version(inputFile) {
def yamlSlurper = new groovy.yaml.YamlSlurper()
def loaded_viash_config = yamlSlurper.parse(file(inputFile))
def version = (loaded_viash_config.version) ? loaded_viash_config.version : "unknown_version"
println("Version to be used: ${version}")
return version
}

12
src/runner/nextflow.config Normal file
View File

@@ -0,0 +1,12 @@
manifest {
nextflowVersion = '!>=20.12.1-edge'
}
process {
withName: publishStatesProc {
publishDir = [ enabled: false ]
}
}
// include common settings
includeConfig("${params.rootDir}/src/config/labels.config")

9
target/executable/io/interop_summary_to_csv/.config.vsh.yaml
View File

@@ -141,9 +141,9 @@ build_info:
output: "target/executable/io/interop_summary_to_csv"
executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -156,7 +156,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

4
target/executable/io/interop_summary_to_csv/interop_summary_to_csv
View File

@@ -470,9 +470,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \
mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/
LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv"
LABEL org.opencontainers.image.created="2024-12-05T10:21:34Z"
LABEL org.opencontainers.image.created="2024-12-11T08:48:31Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="6e6be28b85ab619214ae05a017a33498c0dc8890"
LABEL org.opencontainers.image.revision="64a371e168472c987d9ec61b0373b0e28762dfcd"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

202
target/executable/io/publish/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,202 @@
name: "publish"
namespace: "io"
version: "main"
argument_groups:
- name: "Input arguments"
arguments:
- type: "file"
name: "--input"
description: "Directory to write fastq data to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--input_falco"
description: "Directory to write falco output to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--input_multiqc"
description: "Directory to write falco output to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Output arguments"
arguments:
- type: "file"
name: "--output"
info: null
default:
- "fastq"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output_falco"
info: null
default:
- "qc/fastqc"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output_multiqc"
info: null
default:
- "qc/multiqc_report.html"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "bash_script"
path: "code.sh"
is_executable: true
description: "Publish the processed results of the run"
info: null
status: "enabled"
requirements:
commands:
- "ps"
license: "MIT"
links:
repository: "https://github.com/viash-hub/demultiplex"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "debian:stable-slim"
target_registry: "images.viash-hub.com"
target_tag: "main"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/io/publish/config.vsh.yaml"
runner: "executable"
engine: "docker|native"
output: "target/executable/io/publish"
executable: "target/executable/io/publish/publish"
viash_version: "0.9.0"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
description: "Demultiplexing pipeline\n"
info:
test_resources:
- path: "gs://viash-hub-test-data/demultiplex/v2/"
dest: "testData"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"
keywords:
- "bioinformatics"
- "sequence"
- "demultiplexing"
- "pipeline"
license: "MIT"
organization: "vsh"
links:
repository: "https://github.com/viash-hub/demultiplex"
issue_tracker: "https://github.com/viash-hub/demultiplex/issues"

1234
target/executable/io/publish/publish Executable file
View File

File diff suppressed because it is too large Load Diff

9
target/executable/io/untar/.config.vsh.yaml
View File

@@ -148,9 +148,9 @@ build_info:
output: "target/executable/io/untar"
executable: "target/executable/io/untar/untar"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -163,7 +163,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

4
target/executable/io/untar/untar
View File

@@ -476,9 +476,9 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
LABEL org.opencontainers.image.description="Companion container for running component io untar"
LABEL org.opencontainers.image.created="2024-12-05T10:21:34Z"
LABEL org.opencontainers.image.created="2024-12-11T08:48:32Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="6e6be28b85ab619214ae05a017a33498c0dc8890"
LABEL org.opencontainers.image.revision="64a371e168472c987d9ec61b0373b0e28762dfcd"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

9
target/nextflow/dataflow/combine_samples/.config.vsh.yaml
View File

@@ -139,9 +139,9 @@ build_info:
output: "target/nextflow/dataflow/combine_samples"
executable: "target/nextflow/dataflow/combine_samples/main.nf"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -154,7 +154,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

8
target/nextflow/dataflow/combine_samples/main.nf
View File

@@ -2972,9 +2972,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/combine_samples",
"viash_version" : "0.9.0",
"git_commit" : "6e6be28b85ab619214ae05a017a33498c0dc8890",
"git_remote" : "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-5-g6e6be28"
"git_commit" : "64a371e168472c987d9ec61b0373b0e28762dfcd",
"git_remote" : "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-7-g64a371e"
},
"package_config" : {
"name" : "demultiplex",
@@ -2992,7 +2992,7 @@ meta = [
"source" : "src",
"target" : "target",
"config_mods" : [
".requirements.commands := ['ps']\n",
".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n",
".engines += { type: \\"native\\" }",
".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
".engines[.type == 'docker'].target_tag := 'main'"

9
target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml
View File

@@ -133,9 +133,9 @@ build_info:
output: "target/nextflow/dataflow/gather_fastqs_and_validate"
executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -148,7 +148,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

8
target/nextflow/dataflow/gather_fastqs_and_validate/main.nf
View File

@@ -2965,9 +2965,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/gather_fastqs_and_validate",
"viash_version" : "0.9.0",
"git_commit" : "6e6be28b85ab619214ae05a017a33498c0dc8890",
"git_remote" : "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-5-g6e6be28"
"git_commit" : "64a371e168472c987d9ec61b0373b0e28762dfcd",
"git_remote" : "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-7-g64a371e"
},
"package_config" : {
"name" : "demultiplex",
@@ -2985,7 +2985,7 @@ meta = [
"source" : "src",
"target" : "target",
"config_mods" : [
".requirements.commands := ['ps']\n",
".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n",
".engines += { type: \\"native\\" }",
".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
".engines[.type == 'docker'].target_tag := 'main'"

25
target/nextflow/demultiplex/.config.vsh.yaml
View File

@@ -3,6 +3,14 @@ version: "main"
argument_groups:
- name: "Input arguments"
arguments:
- type: "string"
name: "--id"
description: "Unique identifier for the run"
info: null
required: false
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--input"
description: "Directory containing raw sequencing data"
@@ -45,9 +53,11 @@ argument_groups:
name: "--output"
description: "Directory to write fastq data to"
info: null
default:
- "$id/fastq"
must_exist: true
create_parent: true
required: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
@@ -56,7 +66,7 @@ argument_groups:
description: "Directory to write falco output to"
info: null
default:
- "$id/falco"
- "$id/qc/fastqc"
must_exist: true
create_parent: true
required: false
@@ -68,7 +78,7 @@ argument_groups:
description: "Directory to write falco output to"
info: null
default:
- "$id/multiqc_report.html"
- "$id/qc/multiqc_report.html"
must_exist: true
create_parent: true
required: false
@@ -210,9 +220,9 @@ build_info:
output: "target/nextflow/demultiplex"
executable: "target/nextflow/demultiplex/main.nf"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
dependencies:
- "target/nextflow/io/untar"
- "target/nextflow/dataflow/gather_fastqs_and_validate"
@@ -234,7 +244,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

50
target/nextflow/demultiplex/main.nf
View File

@@ -2810,6 +2810,15 @@ meta = [
{
"name" : "Input arguments",
"arguments" : [
{
"type" : "string",
"name" : "--id",
"description" : "Unique identifier for the run",
"required" : false,
"direction" : "input",
"multiple" : false,
"multiple_sep" : ";"
},
{
"type" : "file",
"name" : "--input",
@@ -2854,9 +2863,12 @@ meta = [
"type" : "file",
"name" : "--output",
"description" : "Directory to write fastq data to",
"default" : [
"$id/fastq"
],
"must_exist" : true,
"create_parent" : true,
"required" : true,
"required" : false,
"direction" : "output",
"multiple" : false,
"multiple_sep" : ";"
@@ -2866,7 +2878,7 @@ meta = [
"name" : "--output_falco",
"description" : "Directory to write falco output to",
"default" : [
"$id/falco"
"$id/qc/fastqc"
],
"must_exist" : true,
"create_parent" : true,
@@ -2880,7 +2892,7 @@ meta = [
"name" : "--output_multiqc",
"description" : "Directory to write falco output to",
"default" : [
"$id/multiqc_report.html"
"$id/qc/multiqc_report.html"
],
"must_exist" : true,
"create_parent" : true,
@@ -3076,9 +3088,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/demultiplex",
"viash_version" : "0.9.0",
"git_commit" : "6e6be28b85ab619214ae05a017a33498c0dc8890",
"git_remote" : "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-5-g6e6be28"
"git_commit" : "64a371e168472c987d9ec61b0373b0e28762dfcd",
"git_remote" : "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-7-g64a371e"
},
"package_config" : {
"name" : "demultiplex",
@@ -3096,7 +3108,7 @@ meta = [
"source" : "src",
"target" : "target",
"config_mods" : [
".requirements.commands := ['ps']\n",
".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n",
".engines += { type: \\"native\\" }",
".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
".engines[.type == 'docker'].target_tag := 'main'"
@@ -3136,6 +3148,7 @@ workflow run_wf {
main:
samples_ch = input_ch
// untar input if needed
| untar.run(
directives: [label: ["lowmem", "lowcpu"]],
@@ -3241,11 +3254,15 @@ workflow run_wf {
| bcl_convert.run(
runIf: {id, state -> state.demultiplexer in ["bclconvert"]},
directives: [label: ["highmem", "midcpu"]],
fromState: [
"bcl_input_directory": "input",
"sample_sheet": "run_information",
"output_directory": "output",
],
fromState: { id, state ->
[
bcl_input_directory: state.input,
sample_sheet: state.run_information,
output_directory: state.output,
reports: "reports",
logs: "logs"
]
},
toState: {id, result, state ->
def toAdd = [
"output_demultiplexer" : result.output_directory,
@@ -3291,6 +3308,8 @@ workflow run_wf {
)
output_ch = samples_ch
| combine_samples.run(
fromState: { id, state ->
[
@@ -3318,10 +3337,10 @@ workflow run_wf {
},
toState: { id, result, state ->
state + [ "output_falco" : result.outdir ]
},
}
)
| multiqc.run(
directives: [label: ["lowcpu", "lowmem"]],
directives: [label: ["midcpu", "midmem"]],
fromState: {id, state ->
def new_state = [
"input": [state.output_falco],
@@ -3338,10 +3357,11 @@ workflow run_wf {
},
toState: { id, result, state ->
state + [ "output_multiqc" : result.output_report ]
},
}
)
| setState(
[
//"_meta": "_meta",
"output": "output_demultiplexer",
"output_falco": "output_falco",
"output_multiqc": "output_multiqc"

14
target/nextflow/demultiplex/nextflow_schema.json
View File

@@ -14,6 +14,16 @@
"properties": {
"id": {
"type":
"string",
"description": "Type: `string`. Unique identifier for the run",
"help_text": "Type: `string`. Unique identifier for the run"
}
,
"input": {
"type":
"string",
@@ -59,8 +69,8 @@
"output": {
"type":
"string",
"description": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write fastq data to",
"help_text": "Type: `file`, required, default: `$id.$key.output.output`. Directory to write fastq data to"
"description": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to",
"help_text": "Type: `file`, default: `$id.$key.output.output`. Directory to write fastq data to"
,
"default":"$id.$key.output.output"
}

9
target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml
View File

@@ -141,9 +141,9 @@ build_info:
output: "target/nextflow/io/interop_summary_to_csv"
executable: "target/nextflow/io/interop_summary_to_csv/main.nf"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -156,7 +156,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

8
target/nextflow/io/interop_summary_to_csv/main.nf
View File

@@ -2977,9 +2977,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/interop_summary_to_csv",
"viash_version" : "0.9.0",
"git_commit" : "6e6be28b85ab619214ae05a017a33498c0dc8890",
"git_remote" : "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-5-g6e6be28"
"git_commit" : "64a371e168472c987d9ec61b0373b0e28762dfcd",
"git_remote" : "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-7-g64a371e"
},
"package_config" : {
"name" : "demultiplex",
@@ -2997,7 +2997,7 @@ meta = [
"source" : "src",
"target" : "target",
"config_mods" : [
".requirements.commands := ['ps']\n",
".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n",
".engines += { type: \\"native\\" }",
".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
".engines[.type == 'docker'].target_tag := 'main'"

202
target/nextflow/io/publish/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,202 @@
name: "publish"
namespace: "io"
version: "main"
argument_groups:
- name: "Input arguments"
arguments:
- type: "file"
name: "--input"
description: "Directory to write fastq data to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--input_falco"
description: "Directory to write falco output to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--input_multiqc"
description: "Directory to write falco output to"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Output arguments"
arguments:
- type: "file"
name: "--output"
info: null
default:
- "fastq"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output_falco"
info: null
default:
- "qc/fastqc"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--output_multiqc"
info: null
default:
- "qc/multiqc_report.html"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "bash_script"
path: "code.sh"
is_executable: true
description: "Publish the processed results of the run"
info: null
status: "enabled"
requirements:
commands:
- "ps"
license: "MIT"
links:
repository: "https://github.com/viash-hub/demultiplex"
runners:
- type: "executable"
id: "executable"
docker_setup_strategy: "ifneedbepullelsecachedbuild"
- type: "nextflow"
id: "nextflow"
directives:
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
debug: false
container: "docker"
engines:
- type: "docker"
id: "docker"
image: "debian:stable-slim"
target_registry: "images.viash-hub.com"
target_tag: "main"
namespace_separator: "/"
setup:
- type: "apt"
packages:
- "procps"
interactive: false
entrypoint: []
cmd: null
- type: "native"
id: "native"
build_info:
config: "src/io/publish/config.vsh.yaml"
runner: "nextflow"
engine: "docker|native"
output: "target/nextflow/io/publish"
executable: "target/nextflow/io/publish/main.nf"
viash_version: "0.9.0"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
description: "Demultiplexing pipeline\n"
info:
test_resources:
- path: "gs://viash-hub-test-data/demultiplex/v2/"
dest: "testData"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"
keywords:
- "bioinformatics"
- "sequence"
- "demultiplexing"
- "pipeline"
license: "MIT"
organization: "vsh"
links:
repository: "https://github.com/viash-hub/demultiplex"
issue_tracker: "https://github.com/viash-hub/demultiplex/issues"

3603
target/nextflow/io/publish/main.nf Normal file
View File

File diff suppressed because it is too large Load Diff

125
target/nextflow/io/publish/nextflow.config Normal file
View File

@@ -0,0 +1,125 @@
manifest {
name = 'io/publish'
mainScript = 'main.nf'
nextflowVersion = '!>=20.12.1-edge'
version = 'main'
description = 'Publish the processed results of the run'
}
process.container = 'nextflow/bash:latest'
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
profiles {
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
docker {
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
singularity {
singularity.enabled = true
singularity.autoMounts = true
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
podman {
podman.enabled = true
docker.enabled = false
singularity.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
shifter {
shifter.enabled = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
charliecloud.enabled = false
}
charliecloud {
charliecloud.enabled = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
}
}
process{
withLabel: mem1gb { memory = 1000000000.B }
withLabel: mem2gb { memory = 2000000000.B }
withLabel: mem5gb { memory = 5000000000.B }
withLabel: mem10gb { memory = 10000000000.B }
withLabel: mem20gb { memory = 20000000000.B }
withLabel: mem50gb { memory = 50000000000.B }
withLabel: mem100gb { memory = 100000000000.B }
withLabel: mem200gb { memory = 200000000000.B }
withLabel: mem500gb { memory = 500000000000.B }
withLabel: mem1tb { memory = 1000000000000.B }
withLabel: mem2tb { memory = 2000000000000.B }
withLabel: mem5tb { memory = 5000000000000.B }
withLabel: mem10tb { memory = 10000000000000.B }
withLabel: mem20tb { memory = 20000000000000.B }
withLabel: mem50tb { memory = 50000000000000.B }
withLabel: mem100tb { memory = 100000000000000.B }
withLabel: mem200tb { memory = 200000000000000.B }
withLabel: mem500tb { memory = 500000000000000.B }
withLabel: mem1gib { memory = 1073741824.B }
withLabel: mem2gib { memory = 2147483648.B }
withLabel: mem4gib { memory = 4294967296.B }
withLabel: mem8gib { memory = 8589934592.B }
withLabel: mem16gib { memory = 17179869184.B }
withLabel: mem32gib { memory = 34359738368.B }
withLabel: mem64gib { memory = 68719476736.B }
withLabel: mem128gib { memory = 137438953472.B }
withLabel: mem256gib { memory = 274877906944.B }
withLabel: mem512gib { memory = 549755813888.B }
withLabel: mem1tib { memory = 1099511627776.B }
withLabel: mem2tib { memory = 2199023255552.B }
withLabel: mem4tib { memory = 4398046511104.B }
withLabel: mem8tib { memory = 8796093022208.B }
withLabel: mem16tib { memory = 17592186044416.B }
withLabel: mem32tib { memory = 35184372088832.B }
withLabel: mem64tib { memory = 70368744177664.B }
withLabel: mem128tib { memory = 140737488355328.B }
withLabel: mem256tib { memory = 281474976710656.B }
withLabel: mem512tib { memory = 562949953421312.B }
withLabel: cpu1 { cpus = 1 }
withLabel: cpu2 { cpus = 2 }
withLabel: cpu5 { cpus = 5 }
withLabel: cpu10 { cpus = 10 }
withLabel: cpu20 { cpus = 20 }
withLabel: cpu50 { cpus = 50 }
withLabel: cpu100 { cpus = 100 }
withLabel: cpu200 { cpus = 200 }
withLabel: cpu500 { cpus = 500 }
withLabel: cpu1000 { cpus = 1000 }
}

137
target/nextflow/io/publish/nextflow_schema.json Normal file
View File

@@ -0,0 +1,137 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"title": "publish",
"description": "Publish the processed results of the run",
"type": "object",
"definitions": {
"input arguments" : {
"title": "Input arguments",
"type": "object",
"description": "No description",
"properties": {
"input": {
"type":
"string",
"description": "Type: `file`, required. Directory to write fastq data to",
"help_text": "Type: `file`, required. Directory to write fastq data to"
}
,
"input_falco": {
"type":
"string",
"description": "Type: `file`, required. Directory to write falco output to",
"help_text": "Type: `file`, required. Directory to write falco output to"
}
,
"input_multiqc": {
"type":
"string",
"description": "Type: `file`, required. Directory to write falco output to",
"help_text": "Type: `file`, required. Directory to write falco output to"
}
}
},
"output arguments" : {
"title": "Output arguments",
"type": "object",
"description": "No description",
"properties": {
"output": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.output.output`. ",
"help_text": "Type: `file`, default: `$id.$key.output.output`. "
,
"default":"$id.$key.output.output"
}
,
"output_falco": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.output_falco.output_falco`. ",
"help_text": "Type: `file`, default: `$id.$key.output_falco.output_falco`. "
,
"default":"$id.$key.output_falco.output_falco"
}
,
"output_multiqc": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.output_multiqc.html`. ",
"help_text": "Type: `file`, default: `$id.$key.output_multiqc.html`. "
,
"default":"$id.$key.output_multiqc.html"
}
}
},
"nextflow input-output arguments" : {
"title": "Nextflow input-output arguments",
"type": "object",
"description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
"properties": {
"publish_dir": {
"type":
"string",
"description": "Type: `string`, required, example: `output/`. Path to an output directory",
"help_text": "Type: `string`, required, example: `output/`. Path to an output directory."
}
,
"param_list": {
"type":
"string",
"description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel",
"help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.",
"hidden": true
}
}
}
},
"allOf": [
{
"$ref": "#/definitions/input arguments"
},
{
"$ref": "#/definitions/output arguments"
},
{
"$ref": "#/definitions/nextflow input-output arguments"
}
]
}

9
target/nextflow/io/untar/.config.vsh.yaml
View File

@@ -148,9 +148,9 @@ build_info:
output: "target/nextflow/io/untar"
executable: "target/nextflow/io/untar/main.nf"
viash_version: "0.9.0"
git_commit: "6e6be28b85ab619214ae05a017a33498c0dc8890"
git_remote: "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-5-g6e6be28"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
package_config:
name: "demultiplex"
version: "main"
@@ -163,7 +163,8 @@ package_config:
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n"
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"

8
target/nextflow/io/untar/main.nf
View File

@@ -2989,9 +2989,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/untar",
"viash_version" : "0.9.0",
"git_commit" : "6e6be28b85ab619214ae05a017a33498c0dc8890",
"git_remote" : "https://x-access-token:ghs_iopCikoFK5KlWGbHHLIwAf82AMtbiI0fzew1@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-5-g6e6be28"
"git_commit" : "64a371e168472c987d9ec61b0373b0e28762dfcd",
"git_remote" : "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-7-g64a371e"
},
"package_config" : {
"name" : "demultiplex",
@@ -3009,7 +3009,7 @@ meta = [
"source" : "src",
"target" : "target",
"config_mods" : [
".requirements.commands := ['ps']\n",
".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag := '$id'\n",
".engines += { type: \\"native\\" }",
".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'",
".engines[.type == 'docker'].target_tag := 'main'"

191
target/nextflow/runner/.config.vsh.yaml Normal file
View File

@@ -0,0 +1,191 @@
name: "runner"
version: "main"
argument_groups:
- name: "Input arguments"
arguments:
- type: "file"
name: "--input"
description: "Base directory of the form `s3:/<bucket>/Sequencing/<Sequencer>/<RunID>/`"
info: null
must_exist: true
create_parent: true
required: true
direction: "input"
multiple: false
multiple_sep: ";"
- name: "Annotation flags"
arguments:
- type: "boolean_true"
name: "--add_date_time"
description: "Add date and time to the output directory name. This is useful\n\
when running the same pipeline multiple times on the same input\ndirectory.\n"
info: null
direction: "input"
- type: "boolean_true"
name: "--add_workflow_id"
description: "Add a workflow identifier to the output directory name.\n"
info: null
direction: "input"
- name: "Output arguments"
arguments:
- type: "file"
name: "--fastq_output"
info: null
default:
- "fastq"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--falco_output"
info: null
default:
- "qc/fastqc"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
- type: "file"
name: "--multiqc_output"
info: null
default:
- "qc/multiqc_report.html"
must_exist: true
create_parent: true
required: false
direction: "output"
multiple: false
multiple_sep: ";"
resources:
- type: "nextflow_script"
path: "main.nf"
is_executable: true
entrypoint: "run_wf"
description: "Runner for demultiplexing of raw sequencing data"
info: null
status: "enabled"
requirements:
commands:
- "ps"
dependencies:
- name: "demultiplex"
repository:
type: "local"
- name: "io/publish"
repository:
type: "local"
license: "MIT"
links:
repository: "https://github.com/viash-hub/demultiplex"
runners:
- type: "nextflow"
id: "nextflow"
directives:
tag: "$id"
auto:
simplifyInput: true
simplifyOutput: false
transcript: false
publish: false
config:
labels:
mem1gb: "memory = 1000000000.B"
mem2gb: "memory = 2000000000.B"
mem5gb: "memory = 5000000000.B"
mem10gb: "memory = 10000000000.B"
mem20gb: "memory = 20000000000.B"
mem50gb: "memory = 50000000000.B"
mem100gb: "memory = 100000000000.B"
mem200gb: "memory = 200000000000.B"
mem500gb: "memory = 500000000000.B"
mem1tb: "memory = 1000000000000.B"
mem2tb: "memory = 2000000000000.B"
mem5tb: "memory = 5000000000000.B"
mem10tb: "memory = 10000000000000.B"
mem20tb: "memory = 20000000000000.B"
mem50tb: "memory = 50000000000000.B"
mem100tb: "memory = 100000000000000.B"
mem200tb: "memory = 200000000000000.B"
mem500tb: "memory = 500000000000000.B"
mem1gib: "memory = 1073741824.B"
mem2gib: "memory = 2147483648.B"
mem4gib: "memory = 4294967296.B"
mem8gib: "memory = 8589934592.B"
mem16gib: "memory = 17179869184.B"
mem32gib: "memory = 34359738368.B"
mem64gib: "memory = 68719476736.B"
mem128gib: "memory = 137438953472.B"
mem256gib: "memory = 274877906944.B"
mem512gib: "memory = 549755813888.B"
mem1tib: "memory = 1099511627776.B"
mem2tib: "memory = 2199023255552.B"
mem4tib: "memory = 4398046511104.B"
mem8tib: "memory = 8796093022208.B"
mem16tib: "memory = 17592186044416.B"
mem32tib: "memory = 35184372088832.B"
mem64tib: "memory = 70368744177664.B"
mem128tib: "memory = 140737488355328.B"
mem256tib: "memory = 281474976710656.B"
mem512tib: "memory = 562949953421312.B"
cpu1: "cpus = 1"
cpu2: "cpus = 2"
cpu5: "cpus = 5"
cpu10: "cpus = 10"
cpu20: "cpus = 20"
cpu50: "cpus = 50"
cpu100: "cpus = 100"
cpu200: "cpus = 200"
cpu500: "cpus = 500"
cpu1000: "cpus = 1000"
debug: false
container: "docker"
engines:
- type: "native"
id: "native"
- type: "native"
id: "native"
build_info:
config: "src/runner/config.vsh.yaml"
runner: "nextflow"
engine: "native|native"
output: "target/nextflow/runner"
executable: "target/nextflow/runner/main.nf"
viash_version: "0.9.0"
git_commit: "64a371e168472c987d9ec61b0373b0e28762dfcd"
git_remote: "https://x-access-token:ghs_NEnBDPGgmRYfKlNL6VeMqiNleTP4aC2s2yz6@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-7-g64a371e"
dependencies:
- "target/nextflow/demultiplex"
- "target/nextflow/io/publish"
package_config:
name: "demultiplex"
version: "main"
description: "Demultiplexing pipeline\n"
info:
test_resources:
- path: "gs://viash-hub-test-data/demultiplex/v2/"
dest: "testData"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:
- ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
\ := '$id'\n"
- ".engines += { type: \"native\" }"
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
- ".engines[.type == 'docker'].target_tag := 'main'"
keywords:
- "bioinformatics"
- "sequence"
- "demultiplexing"
- "pipeline"
license: "MIT"
organization: "vsh"
links:
repository: "https://github.com/viash-hub/demultiplex"
issue_tracker: "https://github.com/viash-hub/demultiplex/issues"

3254
target/nextflow/runner/main.nf Normal file
View File

File diff suppressed because it is too large Load Diff

125
target/nextflow/runner/nextflow.config Normal file
View File

@@ -0,0 +1,125 @@
manifest {
name = 'runner'
mainScript = 'main.nf'
nextflowVersion = '!>=20.12.1-edge'
version = 'main'
description = 'Runner for demultiplexing of raw sequencing data'
}
process.container = 'nextflow/bash:latest'
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
profiles {
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
docker {
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
singularity {
singularity.enabled = true
singularity.autoMounts = true
docker.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
podman {
podman.enabled = true
docker.enabled = false
singularity.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
shifter {
shifter.enabled = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
charliecloud.enabled = false
}
charliecloud {
charliecloud.enabled = true
docker.enabled = false
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
}
}
process{
withLabel: mem1gb { memory = 1000000000.B }
withLabel: mem2gb { memory = 2000000000.B }
withLabel: mem5gb { memory = 5000000000.B }
withLabel: mem10gb { memory = 10000000000.B }
withLabel: mem20gb { memory = 20000000000.B }
withLabel: mem50gb { memory = 50000000000.B }
withLabel: mem100gb { memory = 100000000000.B }
withLabel: mem200gb { memory = 200000000000.B }
withLabel: mem500gb { memory = 500000000000.B }
withLabel: mem1tb { memory = 1000000000000.B }
withLabel: mem2tb { memory = 2000000000000.B }
withLabel: mem5tb { memory = 5000000000000.B }
withLabel: mem10tb { memory = 10000000000000.B }
withLabel: mem20tb { memory = 20000000000000.B }
withLabel: mem50tb { memory = 50000000000000.B }
withLabel: mem100tb { memory = 100000000000000.B }
withLabel: mem200tb { memory = 200000000000000.B }
withLabel: mem500tb { memory = 500000000000000.B }
withLabel: mem1gib { memory = 1073741824.B }
withLabel: mem2gib { memory = 2147483648.B }
withLabel: mem4gib { memory = 4294967296.B }
withLabel: mem8gib { memory = 8589934592.B }
withLabel: mem16gib { memory = 17179869184.B }
withLabel: mem32gib { memory = 34359738368.B }
withLabel: mem64gib { memory = 68719476736.B }
withLabel: mem128gib { memory = 137438953472.B }
withLabel: mem256gib { memory = 274877906944.B }
withLabel: mem512gib { memory = 549755813888.B }
withLabel: mem1tib { memory = 1099511627776.B }
withLabel: mem2tib { memory = 2199023255552.B }
withLabel: mem4tib { memory = 4398046511104.B }
withLabel: mem8tib { memory = 8796093022208.B }
withLabel: mem16tib { memory = 17592186044416.B }
withLabel: mem32tib { memory = 35184372088832.B }
withLabel: mem64tib { memory = 70368744177664.B }
withLabel: mem128tib { memory = 140737488355328.B }
withLabel: mem256tib { memory = 281474976710656.B }
withLabel: mem512tib { memory = 562949953421312.B }
withLabel: cpu1 { cpus = 1 }
withLabel: cpu2 { cpus = 2 }
withLabel: cpu5 { cpus = 5 }
withLabel: cpu10 { cpus = 10 }
withLabel: cpu20 { cpus = 20 }
withLabel: cpu50 { cpus = 50 }
withLabel: cpu100 { cpus = 100 }
withLabel: cpu200 { cpus = 200 }
withLabel: cpu500 { cpus = 500 }
withLabel: cpu1000 { cpus = 1000 }
}

153
target/nextflow/runner/nextflow_schema.json Normal file
View File

@@ -0,0 +1,153 @@
{
"$schema": "http://json-schema.org/draft-07/schema",
"title": "runner",
"description": "Runner for demultiplexing of raw sequencing data",
"type": "object",
"definitions": {
"input arguments" : {
"title": "Input arguments",
"type": "object",
"description": "No description",
"properties": {
"input": {
"type":
"string",
"description": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/`",
"help_text": "Type: `file`, required. Base directory of the form `s3:/\u003cbucket\u003e/Sequencing/\u003cSequencer\u003e/\u003cRunID\u003e/`"
}
}
},
"annotation flags" : {
"title": "Annotation flags",
"type": "object",
"description": "No description",
"properties": {
"add_date_time": {
"type":
"boolean",
"description": "Type: `boolean_true`, default: `false`. Add date and time to the output directory name",
"help_text": "Type: `boolean_true`, default: `false`. Add date and time to the output directory name. This is useful\nwhen running the same pipeline multiple times on the same input\ndirectory.\n"
,
"default":false
}
,
"add_workflow_id": {
"type":
"boolean",
"description": "Type: `boolean_true`, default: `false`. Add a workflow identifier to the output directory name",
"help_text": "Type: `boolean_true`, default: `false`. Add a workflow identifier to the output directory name.\n"
,
"default":false
}
}
},
"output arguments" : {
"title": "Output arguments",
"type": "object",
"description": "No description",
"properties": {
"fastq_output": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. ",
"help_text": "Type: `file`, default: `$id.$key.fastq_output.fastq_output`. "
,
"default":"$id.$key.fastq_output.fastq_output"
}
,
"falco_output": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.falco_output.falco_output`. ",
"help_text": "Type: `file`, default: `$id.$key.falco_output.falco_output`. "
,
"default":"$id.$key.falco_output.falco_output"
}
,
"multiqc_output": {
"type":
"string",
"description": "Type: `file`, default: `$id.$key.multiqc_output.html`. ",
"help_text": "Type: `file`, default: `$id.$key.multiqc_output.html`. "
,
"default":"$id.$key.multiqc_output.html"
}
}
},
"nextflow input-output arguments" : {
"title": "Nextflow input-output arguments",
"type": "object",
"description": "Input/output parameters for Nextflow itself. Please note that both publishDir and publish_dir are supported but at least one has to be configured.",
"properties": {
"publish_dir": {
"type":
"string",
"description": "Type: `string`, required, example: `output/`. Path to an output directory",
"help_text": "Type: `string`, required, example: `output/`. Path to an output directory."
}
,
"param_list": {
"type":
"string",
"description": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel",
"help_text": "Type: `string`, example: `my_params.yaml`. Allows inputting multiple parameter sets to initialise a Nextflow channel. A `param_list` can either be a list of maps, a csv file, a json file, a yaml file, or simply a yaml blob.\n\n* A list of maps (as-is) where the keys of each map corresponds to the arguments of the pipeline. Example: in a `nextflow.config` file: `param_list: [ [\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027], [\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027] ]`.\n* A csv file should have column names which correspond to the different arguments of this pipeline. Example: `--param_list data.csv` with columns `id,input`.\n* A json or a yaml file should be a list of maps, each of which has keys corresponding to the arguments of the pipeline. Example: `--param_list data.json` with contents `[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]`.\n* A yaml blob can also be passed directly as a string. Example: `--param_list \"[ {\u0027id\u0027: \u0027foo\u0027, \u0027input\u0027: \u0027foo.txt\u0027}, {\u0027id\u0027: \u0027bar\u0027, \u0027input\u0027: \u0027bar.txt\u0027} ]\"`.\n\nWhen passing a csv, json or yaml file, relative path names are relativized to the location of the parameter file. No relativation is performed when `param_list` is a list of maps (as-is) or a yaml blob.",
"hidden": true
}
}
}
},
"allOf": [
{
"$ref": "#/definitions/input arguments"
},
{
"$ref": "#/definitions/annotation flags"
},
{
"$ref": "#/definitions/output arguments"
},
{
"$ref": "#/definitions/nextflow input-output arguments"
}
]
}