vsh/demultiplex
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch main with version main (5cb1323)

Browse Source 
Build pipeline: viash-hub.demultiplex.main-6d5fm

Source commit: 5cb13230bf

Source message: Add resource labels to workflows. (#22)

* Add resource labels to workflows.

* Add CHANGELOG entry
This commit is contained in:
CI
2024-11-06 17:52:30 +00:00
parent 0e5d645386
commit 304b1d24b3
19 changed files with 125 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
CHANGELOG.md
View File

@@ -1,3 +1,9 @@
# demultiplex v0.1.2
## Minor updates
* Add resource labels to workflows (PR #21).
# demultiplex v0.1.1
## Minor updates

79
src/config/labels.config
View File

@@ -1,26 +1,81 @@
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
container = 'nextflow/bash:latest'
// default resources
memory = { 8.Gb * task.attempt }
cpus = 8
maxForks = 36
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
maxMemory = 192.GB
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 8 }
withLabel: midcpu { cpus = 16 }
withLabel: highcpu { cpus = 32 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
}
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
local {
// This config is for local processing.
process {
maxMemory = 25.GB
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 6 }
withLabel: highcpu { cpus = 12 }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
}
}
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare

38
src/config/tests.config
View File

@@ -1,38 +0,0 @@
process.container = 'nextflow/bash:latest'
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
}

4
src/demultiplex/integration_tests.sh
View File

@@ -10,8 +10,8 @@ viash ns build --setup cb
nextflow run . \
-main-script src/demultiplex/test.nf \
-profile docker,no_publish \
-profile docker,no_publish,local \
-entry test_wf \
-c src/config/tests.config \
-c src/config/labels.config \
--resources_test https://raw.githubusercontent.com/nf-core/test-datasets/demultiplex/testdata/NovaSeq6000/ \
-resume

5
src/demultiplex/main.nf
View File

@@ -6,6 +6,7 @@ workflow run_wf {
samples_ch = input_ch
// untar input if needed
| untar.run(
directives: [label: ["lowmem", "lowcpu"]],
runIf: {id, state ->
def inputStr = state.input.toString()
inputStr.endsWith(".tar.gz") || \
@@ -38,6 +39,7 @@ workflow run_wf {
}
| interop_summary_to_csv.run(
directives: [label: ["lowmem", "verylowcpu"]],
fromState: [
"input": "input",
],
@@ -48,6 +50,7 @@ workflow run_wf {
)
// run bcl_convert
| bcl_convert.run(
directives: [label: ["highmem", "midcpu"]],
fromState: [
"bcl_input_directory": "input",
"sample_sheet": "sample_sheet",
@@ -89,6 +92,7 @@ workflow run_wf {
]
)
| falco.run(
directives: [label: ["lowcpu", "lowmem"]],
fromState: {id, state ->
reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : []
[
@@ -104,6 +108,7 @@ workflow run_wf {
},
)
| multiqc.run(
directives: [label: ["lowcpu", "lowmem"]],
fromState: {id, state ->
[
"input": [

6
target/executable/io/interop_summary_to_csv/.config.vsh.yaml
View File

@@ -141,9 +141,9 @@ build_info:
output: "target/executable/io/interop_summary_to_csv"
executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

4
target/executable/io/interop_summary_to_csv/interop_summary_to_csv
View File

@@ -470,9 +470,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \
mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/
LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv"
LABEL org.opencontainers.image.created="2024-10-08T12:19:34Z"
LABEL org.opencontainers.image.created="2024-11-06T17:42:24Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
LABEL org.opencontainers.image.revision="5cb13230bf682321226addce896a3015e8864913"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

6
target/executable/io/untar/.config.vsh.yaml
View File

@@ -148,9 +148,9 @@ build_info:
output: "target/executable/io/untar"
executable: "target/executable/io/untar/untar"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

4
target/executable/io/untar/untar
View File

@@ -476,9 +476,9 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
LABEL org.opencontainers.image.description="Companion container for running component io untar"
LABEL org.opencontainers.image.created="2024-10-08T12:19:34Z"
LABEL org.opencontainers.image.created="2024-11-06T17:42:24Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
LABEL org.opencontainers.image.revision="5cb13230bf682321226addce896a3015e8864913"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

6
target/nextflow/dataflow/combine_samples/.config.vsh.yaml
View File

@@ -139,9 +139,9 @@ build_info:
output: "target/nextflow/dataflow/combine_samples"
executable: "target/nextflow/dataflow/combine_samples/main.nf"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

6
target/nextflow/dataflow/combine_samples/main.nf
View File

@@ -2972,9 +2972,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/combine_samples",
"viash_version" : "0.9.0",
"git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e",
"git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-2-ga311a12"
"git_commit" : "5cb13230bf682321226addce896a3015e8864913",
"git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-3-g5cb1323"
},
"package_config" : {
"name" : "demultiplex",

6
target/nextflow/dataflow/gather_fastqs_and_validate/.config.vsh.yaml
View File

@@ -133,9 +133,9 @@ build_info:
output: "target/nextflow/dataflow/gather_fastqs_and_validate"
executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

6
target/nextflow/dataflow/gather_fastqs_and_validate/main.nf
View File

@@ -2965,9 +2965,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/gather_fastqs_and_validate",
"viash_version" : "0.9.0",
"git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e",
"git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-2-ga311a12"
"git_commit" : "5cb13230bf682321226addce896a3015e8864913",
"git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-3-g5cb1323"
},
"package_config" : {
"name" : "demultiplex",

6
target/nextflow/demultiplex/.config.vsh.yaml
View File

@@ -186,9 +186,9 @@ build_info:
output: "target/nextflow/demultiplex"
executable: "target/nextflow/demultiplex/main.nf"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
dependencies:
- "target/nextflow/io/untar"
- "target/nextflow/dataflow/gather_fastqs_and_validate"

11
target/nextflow/demultiplex/main.nf
View File

@@ -3049,9 +3049,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/demultiplex",
"viash_version" : "0.9.0",
"git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e",
"git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-2-ga311a12"
"git_commit" : "5cb13230bf682321226addce896a3015e8864913",
"git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-3-g5cb1323"
},
"package_config" : {
"name" : "demultiplex",
@@ -3110,6 +3110,7 @@ workflow run_wf {
samples_ch = input_ch
// untar input if needed
| untar.run(
directives: [label: ["lowmem", "lowcpu"]],
runIf: {id, state ->
def inputStr = state.input.toString()
inputStr.endsWith(".tar.gz") || \
@@ -3142,6 +3143,7 @@ workflow run_wf {
}
| interop_summary_to_csv.run(
directives: [label: ["lowmem", "verylowcpu"]],
fromState: [
"input": "input",
],
@@ -3152,6 +3154,7 @@ workflow run_wf {
)
// run bcl_convert
| bcl_convert.run(
directives: [label: ["highmem", "midcpu"]],
fromState: [
"bcl_input_directory": "input",
"sample_sheet": "sample_sheet",
@@ -3193,6 +3196,7 @@ workflow run_wf {
]
)
| falco.run(
directives: [label: ["lowcpu", "lowmem"]],
fromState: {id, state ->
reverse_fastqs_list = state.reverse_fastqs ? state.reverse_fastqs : []
[
@@ -3208,6 +3212,7 @@ workflow run_wf {
},
)
| multiqc.run(
directives: [label: ["lowcpu", "lowmem"]],
fromState: {id, state ->
[
"input": [

6
target/nextflow/io/interop_summary_to_csv/.config.vsh.yaml
View File

@@ -141,9 +141,9 @@ build_info:
output: "target/nextflow/io/interop_summary_to_csv"
executable: "target/nextflow/io/interop_summary_to_csv/main.nf"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

6
target/nextflow/io/interop_summary_to_csv/main.nf
View File

@@ -2977,9 +2977,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/interop_summary_to_csv",
"viash_version" : "0.9.0",
"git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e",
"git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-2-ga311a12"
"git_commit" : "5cb13230bf682321226addce896a3015e8864913",
"git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-3-g5cb1323"
},
"package_config" : {
"name" : "demultiplex",

6
target/nextflow/io/untar/.config.vsh.yaml
View File

@@ -148,9 +148,9 @@ build_info:
output: "target/nextflow/io/untar"
executable: "target/nextflow/io/untar/main.nf"
viash_version: "0.9.0"
git_commit: "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e"
git_remote: "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-2-ga311a12"
git_commit: "5cb13230bf682321226addce896a3015e8864913"
git_remote: "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-3-g5cb1323"
package_config:
name: "demultiplex"
version: "main"

6
target/nextflow/io/untar/main.nf
View File

@@ -2989,9 +2989,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/untar",
"viash_version" : "0.9.0",
"git_commit" : "a311a12056e0dbcfa8103ab0a4f6c7350f2f4c4e",
"git_remote" : "https://x-access-token:ghs_I5gscIGc0N3NRaJ8ljNMSCb4w56xXA2xlViM@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-2-ga311a12"
"git_commit" : "5cb13230bf682321226addce896a3015e8864913",
"git_remote" : "https://x-access-token:ghs_x4A4kChpmjRCoWA3E68PwNsGJxIsF40DBeTf@github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-3-g5cb1323"
},
"package_config" : {
"name" : "demultiplex",