Build branch replace_falco_with_fastqc with version replace_falco_with_fastqc (05f865d)

Build pipeline: viash-hub.demultiplex.replace-falco-with-fastqc-wknmt

Source commit: 05f865da66

Source message: Only add publish_dir if it is not defined
This commit is contained in:
CI
2025-05-28 10:03:04 +00:00
parent abcde69149
commit 58fe13de77
22 changed files with 147 additions and 111 deletions

View File

@@ -9,6 +9,17 @@ workflow run_wf {
main:
output_ch = input_ch
| map { id, state ->
// The argument names for this workflow and the demultiplex workflow may overlap
// here, we store a copy in order to make sure to not accidentally overwrite the state.
def new_state = state + [
"fastq_output_workflow": state.fastq_output,
"multiqc_output_workflow": state.multiqc_output,
"sample_qc_output_workflow": state.sample_qc_output,
"demultiplexer_logs_workflow": state.demultiplexer_logs,
]
return [id, new_state]
}
// Extract the ID from the input.
// If the input is a tarball, strip the suffix.
| map{ id, state ->
@@ -45,17 +56,16 @@ workflow run_wf {
def id1 = (state.plain_output) ? id : "${state.run_id}/${date}"
def id2 = (state.plain_output) ? id : "${id1}_demultiplex_${version}"
def fastq_output_1 = (id2 == "run") ? state.fastq_output : "${id2}/" + state.fastq_output
def sample_qc_output_1 = (id2 == "run") ? state.sample_qc_output : "${id2}/" + state.sample_qc_output
def multiqc_output_1 = (id2 == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output
def run_information_output_1 = (id2 == "run") ? "${state.output_run_information.getName()}" : "${id2}/${state.output_run_information.getName()}"
def demultiplexer_logs_output = (id2 == "run") ? state.demultiplexer_logs : "${id2}/${state.demultiplexer_logs.getName()}"
def prefix = (id2 == "run") ? "" : "${id2}/"
// These output names are determined by arguments.
def fastq_output_1 = "${prefix}${state.fastq_output_workflow}"
def sample_qc_output_1 = "${prefix}${state.sample_qc_output_workflow}"
def multiqc_output_1 = "${prefix}${state.multiqc_output_workflow}"
def demultiplexer_logs_output = "${prefix}${state.demultiplexer_logs_workflow}"
// The name of the output file for the run information is determined by the input file name.
def run_information_output_1 = "${prefix}${state.output_run_information.getName()}"
if (id2 == "run") {
println("Publising to ${params.publish_dir}")
} else {
println("Publising to ${params.publish_dir}/${id2}")
}
println("Publising to ${params.publish_dir}/${prefix}")
[
input: state.output,

View File

@@ -1,25 +1,31 @@
import java.nio.file.Files
import nextflow.exception.WorkflowScriptErrorException
// Create temporary directory
def tempDir = Files.createTempDirectory("demultiplex_runner_integration_test")
println "Created temp directory: $tempDir"
// Register shutdown hook to delete it on JVM exit
Runtime.runtime.addShutdownHook(new Thread({
try {
// Delete directory recursively
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
.forEach { Files.delete(it) }
println "Deleted temp directory: $tempDir"
} catch (Exception e) {
println "Failed to delete temp directory: $e"
}
}))
// Create temporary directory for the publish_dir if it is not defined
if (!params.publish_dir && params.publishDir) {
params.publish_dir = params.publishDir
}
if (!params.publish_dir) {
def tempDir = Files.createTempDirectory("demultiplex_runner_integration_test")
println "Created temp directory: $tempDir"
// Register shutdown hook to delete it on JVM exit
Runtime.runtime.addShutdownHook(new Thread({
try {
// Delete directory recursively
Files.walk(tempDir)
.sorted(Comparator.reverseOrder())
.forEach { Files.delete(it) }
println "Deleted temp directory: $tempDir"
} catch (Exception e) {
println "Failed to delete temp directory: $e"
}
}))
params.publish_dir = tempDir
}
// The module inherits the parameters defined before the include statement,
// therefore any parameters set afterwards will not be used by the module.
params.publish_dir = tempDir
include { runner } from params.rootDir + "/target/nextflow/runner/main.nf"
params.resources_test = params.rootDir + "/testData/"
@@ -35,41 +41,51 @@ workflow test {
fromState: {id, state -> state }
)
workflow.onComplete = {
def all_files = file("${tempDir}/200624_A00834_0183_BHMTFYDRXX").listFiles()
assert all_files.size() == 1
def publish_dir = file(all_files[0])
assert publish_dir.name.endsWith("_demultiplex_unknown_version")
def published_items = publish_dir.listFiles()
assert published_items.size() == 4
assert published_items.collect{it.name}.toSet() == ["demultiplexer_logs", "fastq", "qc", "SampleSheet.csv"].toSet()
def fastqc_files = publish_dir.resolve("qc/fastqc").listFiles()
assert fastqc_files.collect{it.name}.toSet() == [
"Sample1_S1_L001_R1_001_fastqc_data.txt",
"Sample1_S1_L001_R1_001_fastqc_report.html",
"Sample1_S1_L001_R1_001_summary.txt",
"Sample23_S3_L001_R1_001_fastqc_data.txt",
"Sample23_S3_L001_R1_001_fastqc_report.html",
"Sample23_S3_L001_R1_001_summary.txt",
"SampleA_S2_L001_R1_001_fastqc_data.txt",
"SampleA_S2_L001_R1_001_fastqc_report.html",
"SampleA_S2_L001_R1_001_summary.txt",
"sampletest_S4_L001_R1_001_fastqc_data.txt",
"sampletest_S4_L001_R1_001_fastqc_report.html",
"sampletest_S4_L001_R1_001_summary.txt",
"Undetermined_S0_L001_R1_001_fastqc_data.txt",
"Undetermined_S0_L001_R1_001_fastqc_report.html",
"Undetermined_S0_L001_R1_001_summary.txt"
].toSet()
assert publish_dir.resolve("qc/multiqc_report.html").exists()
def fastq_files = publish_dir.resolve("fastq").listFiles()
fastq_files.collect{it.name}.toSet() == [
"Sample1_S1_L001_R1_001.fastq.gz",
"Sample23_S3_L001_R1_001.fastq.gz",
"SampleA_S2_L001_R1_001.fastq.gz",
"sampletest_S4_L001_R1_001.fastq.gz",
"Undetermined_S0_L001_R1_001.fastq.gz"
].toSet()
assert publish_dir.resolve("SampleSheet.csv").exists()
try {
// Nexflow only allows exceptions generated using the 'error' function (which throws WorkflowScriptErrorException).
// So in order for the assert statement to work (or allow other errors to let the tests to fail)
// We need to wrap these in WorkflowScriptErrorException. See https://github.com/nextflow-io/nextflow/pull/4458/files
// The error message will show up in .nextflow.log
def publish_subdir = file("${params.publish_dir}/200624_A00834_0183_BHMTFYDRXX")
assert publish_subdir.isDirectory()
def all_files = publish_subdir.listFiles()
assert all_files.size() == 1
def publish_dir = file(all_files[0])
assert publish_dir.name.endsWith("_demultiplex_unknown_version")
def published_items = publish_dir.listFiles()
assert published_items.size() == 4
assert published_items.collect{it.name}.toSet() == ["demultiplexer_logs", "fastq", "qc", "SampleSheet.csv"].toSet()
def fastqc_files = publish_dir.resolve("qc/fastqc").listFiles()
assert fastqc_files.collect{it.name}.toSet() == [
"Sample1_S1_L001_R1_001_fastqc_data.txt",
"Sample1_S1_L001_R1_001_fastqc_report.html",
"Sample1_S1_L001_R1_001_summary.txt",
"Sample23_S3_L001_R1_001_fastqc_data.txt",
"Sample23_S3_L001_R1_001_fastqc_report.html",
"Sample23_S3_L001_R1_001_summary.txt",
"SampleA_S2_L001_R1_001_fastqc_data.txt",
"SampleA_S2_L001_R1_001_fastqc_report.html",
"SampleA_S2_L001_R1_001_summary.txt",
"sampletest_S4_L001_R1_001_fastqc_data.txt",
"sampletest_S4_L001_R1_001_fastqc_report.html",
"sampletest_S4_L001_R1_001_summary.txt",
"Undetermined_S0_L001_R1_001_fastqc_data.txt",
"Undetermined_S0_L001_R1_001_fastqc_report.html",
"Undetermined_S0_L001_R1_001_summary.txt"
].toSet()
assert publish_dir.resolve("qc/multiqc_report.html").exists()
def fastq_files = publish_dir.resolve("fastq").listFiles()
assert fastq_files.collect{it.name}.toSet() == [
"Sample1_S1_L001_R1_001.fastq.gz",
"Sample23_S3_L001_R1_001.fastq.gz",
"SampleA_S2_L001_R1_001.fastq.gz",
"sampletest_S4_L001_R1_001.fastq.gz",
"Undetermined_S0_L001_R1_001.fastq.gz"
].toSet()
assert publish_dir.resolve("SampleSheet.csv").exists()
} catch (Exception e) {
throw new WorkflowScriptErrorException("Integration test failed!", e)
}
}
}

View File

@@ -157,9 +157,9 @@ build_info:
output: "target/executable/io/interop_summary_to_csv"
executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -454,9 +454,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \
mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/
LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv"
LABEL org.opencontainers.image.created="2025-05-27T20:00:48Z"
LABEL org.opencontainers.image.created="2025-05-28T09:38:56Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="b964c6be513b030c07afa48bfd3e4e6bb80644ed"
LABEL org.opencontainers.image.revision="05f865da660a3eb3ada0d4a15c515afdcfe8c087"
LABEL org.opencontainers.image.version="replace_falco_with_fastqc"
VIASHDOCKER

View File

@@ -219,9 +219,9 @@ build_info:
output: "target/executable/io/publish"
executable: "target/executable/io/publish/publish"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -450,9 +450,9 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
LABEL org.opencontainers.image.description="Companion container for running component io publish"
LABEL org.opencontainers.image.created="2025-05-27T20:00:48Z"
LABEL org.opencontainers.image.created="2025-05-28T09:38:55Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="b964c6be513b030c07afa48bfd3e4e6bb80644ed"
LABEL org.opencontainers.image.revision="05f865da660a3eb3ada0d4a15c515afdcfe8c087"
LABEL org.opencontainers.image.version="replace_falco_with_fastqc"
VIASHDOCKER

View File

@@ -156,9 +156,9 @@ build_info:
output: "target/executable/io/untar"
executable: "target/executable/io/untar/untar"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -450,9 +450,9 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
LABEL org.opencontainers.image.description="Companion container for running component io untar"
LABEL org.opencontainers.image.created="2025-05-27T20:00:48Z"
LABEL org.opencontainers.image.created="2025-05-28T09:38:55Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
LABEL org.opencontainers.image.revision="b964c6be513b030c07afa48bfd3e4e6bb80644ed"
LABEL org.opencontainers.image.revision="05f865da660a3eb3ada0d4a15c515afdcfe8c087"
LABEL org.opencontainers.image.version="replace_falco_with_fastqc"
VIASHDOCKER

View File

@@ -165,9 +165,9 @@ build_info:
output: "target/nextflow/dataflow/combine_samples"
executable: "target/nextflow/dataflow/combine_samples/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -3230,9 +3230,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/combine_samples",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -156,9 +156,9 @@ build_info:
output: "target/nextflow/dataflow/gather_fastqs_and_validate"
executable: "target/nextflow/dataflow/gather_fastqs_and_validate/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -3227,9 +3227,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/dataflow/gather_fastqs_and_validate",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -262,9 +262,9 @@ build_info:
output: "target/nextflow/demultiplex"
executable: "target/nextflow/demultiplex/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
dependencies:
- "target/nextflow/io/untar"
- "target/nextflow/dataflow/gather_fastqs_and_validate"

View File

@@ -3369,9 +3369,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/demultiplex",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -157,9 +157,9 @@ build_info:
output: "target/nextflow/io/interop_summary_to_csv"
executable: "target/nextflow/io/interop_summary_to_csv/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -3228,9 +3228,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/interop_summary_to_csv",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -219,9 +219,9 @@ build_info:
output: "target/nextflow/io/publish"
executable: "target/nextflow/io/publish/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -3297,9 +3297,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/publish",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -156,9 +156,9 @@ build_info:
output: "target/nextflow/io/untar"
executable: "target/nextflow/io/untar/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
package_config:
name: "demultiplex"
version: "replace_falco_with_fastqc"

View File

@@ -3227,9 +3227,9 @@ meta = [
"engine" : "docker|native",
"output" : "target/nextflow/io/untar",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",

View File

@@ -211,9 +211,9 @@ build_info:
output: "target/nextflow/runner"
executable: "target/nextflow/runner/main.nf"
viash_version: "0.9.4"
git_commit: "b964c6be513b030c07afa48bfd3e4e6bb80644ed"
git_commit: "05f865da660a3eb3ada0d4a15c515afdcfe8c087"
git_remote: "https://github.com/viash-hub/demultiplex"
git_tag: "v0.1.1-35-gb964c6b"
git_tag: "v0.1.1-39-g05f865d"
dependencies:
- "target/nextflow/demultiplex"
- "target/nextflow/io/publish"

View File

@@ -3291,9 +3291,9 @@ meta = [
"engine" : "native|native",
"output" : "target/nextflow/runner",
"viash_version" : "0.9.4",
"git_commit" : "b964c6be513b030c07afa48bfd3e4e6bb80644ed",
"git_commit" : "05f865da660a3eb3ada0d4a15c515afdcfe8c087",
"git_remote" : "https://github.com/viash-hub/demultiplex",
"git_tag" : "v0.1.1-35-gb964c6b"
"git_tag" : "v0.1.1-39-g05f865d"
},
"package_config" : {
"name" : "demultiplex",
@@ -3350,6 +3350,17 @@ workflow run_wf {
main:
output_ch = input_ch
| map { id, state ->
// The argument names for this workflow and the demultiplex workflow may overlap
// here, we store a copy in order to make sure to not accidentally overwrite the state.
def new_state = state + [
"fastq_output_workflow": state.fastq_output,
"multiqc_output_workflow": state.multiqc_output,
"sample_qc_output_workflow": state.sample_qc_output,
"demultiplexer_logs_workflow": state.demultiplexer_logs,
]
return [id, new_state]
}
// Extract the ID from the input.
// If the input is a tarball, strip the suffix.
| map{ id, state ->
@@ -3386,17 +3397,16 @@ workflow run_wf {
def id1 = (state.plain_output) ? id : "${state.run_id}/${date}"
def id2 = (state.plain_output) ? id : "${id1}_demultiplex_${version}"
def fastq_output_1 = (id2 == "run") ? state.fastq_output : "${id2}/" + state.fastq_output
def sample_qc_output_1 = (id2 == "run") ? state.sample_qc_output : "${id2}/" + state.sample_qc_output
def multiqc_output_1 = (id2 == "run") ? state.multiqc_output : "${id2}/" + state.multiqc_output
def run_information_output_1 = (id2 == "run") ? "${state.output_run_information.getName()}" : "${id2}/${state.output_run_information.getName()}"
def demultiplexer_logs_output = (id2 == "run") ? state.demultiplexer_logs : "${id2}/${state.demultiplexer_logs.getName()}"
def prefix = (id2 == "run") ? "" : "${id2}/"
// These output names are determined by arguments.
def fastq_output_1 = "${prefix}${state.fastq_output_workflow}"
def sample_qc_output_1 = "${prefix}${state.sample_qc_output_workflow}"
def multiqc_output_1 = "${prefix}${state.multiqc_output_workflow}"
def demultiplexer_logs_output = "${prefix}${state.demultiplexer_logs_workflow}"
// The name of the output file for the run information is determined by the input file name.
def run_information_output_1 = "${prefix}${state.output_run_information.getName()}"
if (id2 == "run") {
println("Publising to ${params.publish_dir}")
} else {
println("Publising to ${params.publish_dir}/${id2}")
}
println("Publising to ${params.publish_dir}/${prefix}")
[
input: state.output,