vsh/htrnaseq
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build branch main with version main (65dd41d)

Browse Source 
Build pipeline: viash-hub.htrnaseq.main-vhms8

Source commit: 65dd41d8b1

Source message: Optimize spawning of processes
This commit is contained in:
CI
2024-11-05 17:26:35 +00:00
parent b8abf8c490
commit f2ff92c6ac
127 changed files with 29560 additions and 1004 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
target/executable/stats/combine_star_logs/.config.vsh.yaml
View File

@@ -8,7 +8,7 @@ argument_groups:
name: "--barcodes"
description: "Barcodes responding to the respective log files.\n"
info: null
required: false
required: true
direction: "input"
multiple: true
multiple_sep: ";"
@@ -20,7 +20,7 @@ argument_groups:
- "Log.final.out"
must_exist: true
create_parent: true
required: false
required: true
direction: "input"
multiple: true
multiple_sep: ";"
@@ -34,7 +34,7 @@ argument_groups:
- "Summary.txt"
must_exist: true
create_parent: true
required: false
required: true
direction: "input"
multiple: true
multiple_sep: ";"
@@ -44,7 +44,7 @@ argument_groups:
info: null
must_exist: true
create_parent: true
required: false
required: true
direction: "input"
multiple: true
multiple_sep: ";"
@@ -182,15 +182,18 @@ build_info:
engine: "docker|native"
output: "target/executable/stats/combine_star_logs"
executable: "target/executable/stats/combine_star_logs/combine_star_logs"
viash_version: "0.9.0-RC7"
git_commit: "cf9797232db1306bfd5696287928cababe317d99"
git_remote: "https://x-access-token:ghs_KjB7pWu8DQM3iFulLu7RI06qnt5K8S1A0eaE@github.com/viash-hub/htrnaseq"
viash_version: "0.9.0"
git_commit: "65dd41d8b1b4a307735c72320c96c0880c75f17f"
git_remote: "https://x-access-token:ghs_McZDF0yobnnHmOEb2Q4JaaB3pzr9mz1VbIOs@github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
info:
test_resources:
- path: "gs://viash-hub-test-data/htrnaseq/v1/"
dest: "resources_test"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:

94
target/executable/stats/combine_star_logs/combine_star_logs
View File

@@ -2,9 +2,9 @@
# combine_star_logs main
#
# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a
# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from
# Data Intuitive.
# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
# The component may contain files which fall under a different license. The
# authors of this component should specify the license in the header of such
@@ -175,16 +175,16 @@ function ViashHelp {
echo ""
echo "Arguments:"
echo " --barcodes"
echo " type: string, multiple values allowed"
echo " type: string, required parameter, multiple values allowed"
echo " Barcodes responding to the respective log files."
echo ""
echo " --star_logs"
echo " type: file, multiple values allowed, file must exist"
echo " type: file, required parameter, multiple values allowed, file must exist"
echo " example: Log.final.out"
echo " Paths to the STAR log files (most frequently called Log.final.out)"
echo ""
echo " --gene_summary_logs"
echo " type: file, multiple values allowed, file must exist"
echo " type: file, required parameter, multiple values allowed, file must exist"
echo " example: Summary.txt"
echo " Paths to the Summary.csv files from the STAR Solo output. Can be found"
echo " in"
@@ -192,7 +192,7 @@ function ViashHelp {
echo " directory."
echo ""
echo " --reads_per_gene_logs"
echo " type: file, multiple values allowed, file must exist"
echo " type: file, required parameter, multiple values allowed, file must exist"
echo " Paths to the 'ReadsPerGene.out.tab' files as output by STAR."
echo ""
echo " --output"
@@ -486,9 +486,9 @@ RUN pip install --upgrade pip && \
pip install --upgrade --no-cache-dir "pandas"
LABEL org.opencontainers.image.description="Companion container for running component stats combine_star_logs"
LABEL org.opencontainers.image.created="2024-09-17T08:52:49Z"
LABEL org.opencontainers.image.created="2024-11-05T15:39:41Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
LABEL org.opencontainers.image.revision="cf9797232db1306bfd5696287928cababe317d99"
LABEL org.opencontainers.image.revision="65dd41d8b1b4a307735c72320c96c0880c75f17f"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER
@@ -879,6 +879,22 @@ fi
# check whether required parameters exist
if [ -z ${VIASH_PAR_BARCODES+x} ]; then
ViashError '--barcodes' is a required argument. Use "--help" to get more information on the parameters.
exit 1
fi
if [ -z ${VIASH_PAR_STAR_LOGS+x} ]; then
ViashError '--star_logs' is a required argument. Use "--help" to get more information on the parameters.
exit 1
fi
if [ -z ${VIASH_PAR_GENE_SUMMARY_LOGS+x} ]; then
ViashError '--gene_summary_logs' is a required argument. Use "--help" to get more information on the parameters.
exit 1
fi
if [ -z ${VIASH_PAR_READS_PER_GENE_LOGS+x} ]; then
ViashError '--reads_per_gene_logs' is a required argument. Use "--help" to get more information on the parameters.
exit 1
fi
if [ -z ${VIASH_META_NAME+x} ]; then
ViashError 'name' is a required argument. Use "--help" to get more information on the parameters.
exit 1
@@ -1145,7 +1161,6 @@ trap interrupt INT SIGINT
cat > "\$tempscript" << 'VIASHMAIN'
import logging
import pandas as pd
import numpy as np
from itertools import batched, starmap
### VIASH START
@@ -1192,7 +1207,7 @@ logger.setLevel(logging.DEBUG)
def handle_percentages(column_value):
# TODO: handle this more gracefully
if column_value:
return np.float64(column_value.strip('%'))
return column_value.strip('%')
return column_value
def star_log_to_dataframe(barcode: str, log_path) -> pd.DataFrame:
@@ -1210,7 +1225,7 @@ def summary_to_dataframe(barcode: str, summary_path) -> pd.DataFrame:
logger.info("Reading summary log %s for barcode %s", summary_path, barcode)
result = pd.read_table(summary_path, sep=",",
header=None, names=["Category", "Value"],
index_col=0)
index_col=0, dtype=pd.StringDtype())
logger.info("Read %d row(s) and %d column(s) from summary file at %s",
*result.shape, summary_path)
return result
@@ -1219,9 +1234,14 @@ def summary_to_dataframe(barcode: str, summary_path) -> pd.DataFrame:
def reads_per_gene_to_dataframe(barcode, read_per_gene_path) -> pd.DataFrame:
logger.info("Reading reads per gene file %s for barcode %s", read_per_gene_path, barcode)
result = pd.read_table(read_per_gene_path, skiprows=[0, 1, 2, 3], header=None, sep="\\t",
dtype={"geneID": pd.StringDtype(),
"Unstranded": pd.Int64Dtype(),
"posStrand": pd.Int64Dtype(),
"negStrand": pd.Int64Dtype()},
index_col=0, names=["geneID", "Unstranded", "posStrand", "negStrand"])
result = result[["Unstranded"]] # Do not use .loc here because we need a DataFrame, not a Series
df = pd.DataFrame({"Value": result.sum()})
df = df.rename({"Unstranded": "NumberOfCountedReads"}, errors="raise")
df.index.name = "Category"
logger.info("Read %d row(s) and %d column(s) from reads per gene file at %s",
*df.shape, read_per_gene_path)
@@ -1250,12 +1270,16 @@ def star_log_remove_unwanted_entries_and_adjust_format(barcode, df: pd.DataFrame
"\\n\\t".join(to_keep[~to_keep].index.to_list()))
result = df.loc[to_keep]
# Replace % by pect, remove columns, use camel case and remove spaces
# You might be tempted to use .title() to make everything uppercase,
# but characters which are already uppercase should stay that way.
# (example: NumberOfUMIs and not NumberOfUmis)
result.index = result.index.str.replace("%", "pect")\\
.str.replace(":", "")\\
.str.replace(r"(?:^|\\s).", lambda m:m.group(0).upper(), regex=True)\\
.str.replace(" ", "")
result = result.rename({"UniquelyMappedReadsNumber": "NumberOfMappedReads",
"UniquelyMappedReadsPect": "pctMappedReads"}, errors="raise")
"UniquelyMappedReadsPect": "PctMappedReads"}, errors="raise")
logger.info("Done filtering STAR logs for barcode %s. Result has %d row(s) and %d column(s). "
"Found entries:\\n\\t%s",
barcode, *result.shape, "\\n\\t".join(result.index.to_list()))
@@ -1271,13 +1295,9 @@ def summary_remove_unwanted_entries_and_adjust_format(barcode, df: pd.DataFrame)
"Reads Mapped to Genome: Unique",
"Reads Mapped to Transcriptome: Unique Genes",
"Reads in Cells Mapped to Unique Genes",
"Mean Reads per Cell",
"Median UMI per Cell",
"Median Genes per Cell",
"Q30 Bases in CB+UMI",
"Reads Mapped to Genome: Unique+Multiple",
"Reads Mapped to Transcriptome: Unique+Multipe Genes",
"Fraction of Reads in Cells",
"Median Reads per Cell",
"Mean UMI per Cell",
"Mean Genes per Cell",
@@ -1290,8 +1310,15 @@ def summary_remove_unwanted_entries_and_adjust_format(barcode, df: pd.DataFrame)
result = df.loc[to_keep]
result.index = result.index.str.replace(r"(?:^|\\s).", lambda m:m.group(0).upper(),
regex=True).str.replace(" ", "")
result = result.rename({"UMIsInCells": "NumberOfUMIs",
"TotalGenesDetected": "NumberOfGenes"}, errors="raise")
to_rename = {"UMIsInCells": "NumberOfUMIs",
"TotalGenesDetected": "NumberOfGenes"}
try:
result = result.rename(to_rename, errors="raise")
except KeyError as e:
raise KeyError(f"Tried to rename log entries ({','.join(to_rename)}) in the summary "
f"log for barcode {barcode}, but an entry was not found in the file. "
"Make sure that you are using the correct version of STAR."
f"Available entries: {", ".join(result.index.to_list())}") from e
logger.info("Done filtering summary logs for barcode %s. Result has %d row(s) and %d column(s). "
"Found entries:\\n\\t%s",
barcode, *result.shape, "\\n\\t".join(result.index.to_list()))
@@ -1340,13 +1367,40 @@ def main(par):
all_stats = pd.concat(all_logs_data, axis=1)
logger.info("Log statistics were gathered for the following barcodes: %s",
", ".join(all_stats.index.to_list()))
dtypes = {
'NumberOfInputReads': pd.UInt64Dtype(),
'NumberOfMappedReads': pd.UInt64Dtype(),
'PctMappedReads': pd.Float64Dtype(),
'NumberOfReadsMappedToMultipleLoci': pd.UInt64Dtype(),
'PectOfReadsMappedToMultipleLoci': pd.Float64Dtype(),
'NumberOfReadsMappedToTooManyLoci': pd.UInt64Dtype(),
'PectOfReadsMappedToTooManyLoci': pd.Float64Dtype(),
'NumberOfReadsUnmappedTooManyMismatches': pd.UInt64Dtype(),
'PectOfReadsUnmappedTooManyMismatches': pd.Float64Dtype(),
'NumberOfReadsUnmappedTooShort': pd.UInt64Dtype(),
'PectOfReadsUnmappedTooShort': pd.Float64Dtype(),
'NumberOfReadsUnmappedOther': pd.UInt64Dtype(),
'PectOfReadsUnmappedOther': pd.Float64Dtype(),
'ReadsWithValidBarcodes': pd.Float64Dtype(),
'SequencingSaturation': pd.Float64Dtype(),
'Q30BasesInCB+UMI': pd.Float64Dtype(),
'ReadsMappedToTranscriptome:Unique+MultipeGenes': pd.Float64Dtype(),
'EstimatedNumberOfCells': pd.UInt64Dtype(),
'FractionOfReadsInCells': pd.Float64Dtype(),
'MeanReadsPerCell': pd.UInt64Dtype(),
'NumberOfUMIs': pd.UInt64Dtype(),
'NumberOfGenes': pd.UInt64Dtype(),
'NumberOfCountedReads': pd.UInt64Dtype(),
}
all_stats = all_stats.astype(dtypes)
# batched() is used here to print a limited amount of columnns at a time
# to make sure that they are all displayed (pandas might limit the view for readability)
logger.info("Summary of final output:\\n%s\\n",
"\\n".join(repr(all_stats.loc[:,columns].describe())
for columns in batched(all_stats.columns, 3)))
logger.info("Writing output to %s", par["output"])
all_stats.reset_index("WellBC").to_csv(par["output"], sep="\\t", header=True, index=False)
all_stats.reset_index("WellBC").to_csv(par["output"], sep="\\t", header=True,
index=False, float_format='%g')
logger.info("Finished %s.", meta["name"])
if __name__ == "__main__":

86
target/executable/stats/combine_star_logs/nextflow_labels.config
View File

@@ -1,26 +1,88 @@
executor {
$k8s {
submitRateLimit = '10sec'
pollInterval = '1 sec'
}
}
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
container = 'nextflow/bash:latest'
// default resources
memory = { 8.Gb * task.attempt }
cpus = 8
maxForks = 36
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
maxMemory = 192.GB
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 8 }
withLabel: midcpu { cpus = 16 }
withLabel: highcpu { cpus = 32 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
}
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
local {
// This config is for local processing.
process {
maxMemory = 25.GB
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 6 }
withLabel: highcpu { cpus = 12 }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
}
}
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare

15
target/executable/stats/generate_pool_statistics/.config.vsh.yaml
View File

@@ -124,7 +124,7 @@ runners:
engines:
- type: "docker"
id: "docker"
image: "python:3.11-slim"
image: "python:3.12-slim"
target_registry: "images.viash-hub.com"
target_tag: "main"
namespace_separator: "/"
@@ -154,15 +154,18 @@ build_info:
engine: "docker|native"
output: "target/executable/stats/generate_pool_statistics"
executable: "target/executable/stats/generate_pool_statistics/generate_pool_statistics"
viash_version: "0.9.0-RC7"
git_commit: "cf9797232db1306bfd5696287928cababe317d99"
git_remote: "https://x-access-token:ghs_KjB7pWu8DQM3iFulLu7RI06qnt5K8S1A0eaE@github.com/viash-hub/htrnaseq"
viash_version: "0.9.0"
git_commit: "65dd41d8b1b4a307735c72320c96c0880c75f17f"
git_remote: "https://x-access-token:ghs_McZDF0yobnnHmOEb2Q4JaaB3pzr9mz1VbIOs@github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
info:
test_resources:
- path: "gs://viash-hub-test-data/htrnaseq/v1/"
dest: "resources_test"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:

44
target/executable/stats/generate_pool_statistics/generate_pool_statistics
View File

@@ -2,9 +2,9 @@
# generate_pool_statistics main
#
# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a
# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from
# Data Intuitive.
# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
# The component may contain files which fall under a different license. The
# authors of this component should specify the license in the header of such
@@ -468,7 +468,7 @@ function ViashDockerfile {
if [[ "$engine_id" == "docker" ]]; then
cat << 'VIASHDOCKER'
FROM python:3.11-slim
FROM python:3.12-slim
ENTRYPOINT []
RUN apt-get update && \
DEBIAN_FRONTEND=noninteractive apt-get install -y procps && \
@@ -478,9 +478,9 @@ RUN pip install --upgrade pip && \
pip install --upgrade --no-cache-dir "pandas"
LABEL org.opencontainers.image.description="Companion container for running component stats generate_pool_statistics"
LABEL org.opencontainers.image.created="2024-09-17T08:52:49Z"
LABEL org.opencontainers.image.created="2024-11-05T15:39:43Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
LABEL org.opencontainers.image.revision="cf9797232db1306bfd5696287928cababe317d99"
LABEL org.opencontainers.image.revision="65dd41d8b1b4a307735c72320c96c0880c75f17f"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER
@@ -1080,9 +1080,13 @@ if __name__ == "__main__":
nr_reads_nr_genes_wells = []
for nr_reads_nr_genes_file in par["nrReadsNrGenesPerChrom"]:
nr_reads_nr_genes_wells.append(pd.read_csv(nr_reads_nr_genes_file,
header=0, delimiter="\\t"))
nr_reads_nr_genes_pool = pd.concat(nr_reads_nr_genes_wells, ignore_index=True)
total_nr_reads_per_chromosome = nr_reads_nr_genes_pool.pivot_table(index="WellBC", columns="Chr",
header=0, delimiter="\\t",
dtype={"WellBC": pd.StringDtype(),
"Chr": pd.StringDtype(),
"NumberOfReads": pd.UInt64Dtype(),
"NumberOfGenes": pd.UInt64Dtype()}))
nr_reads_nr_genes_pool = pd.concat(nr_reads_nr_genes_wells, ignore_index=True,)
total_nr_reads_per_chromosome = nr_reads_nr_genes_pool.pivot_table(index="WellBC", columns="Chr",
values=["NumberOfReads"], fill_value=0,
aggfunc="sum").droplevel(0, axis=1)
total_nr_reads_per_chromosome.columns.name = None
@@ -1123,19 +1127,25 @@ if __name__ == "__main__":
total_chromosomal_reads = total_nr_reads_per_chromosome.loc[:,matching_chromosomes].sum(axis=1)
percentage_chromosomal_reads = round(total_chromosomal_reads / total_sum_of_reads * 100, 2)
cols_to_add = {
"pctChrom": percentage_chromosomal_reads,
"pctMT": percentage_mitochondrial_reads,
"pctERCC": percentage_ercc_reads,
"SumReads": total_sum_of_reads,
"NumberOfGenes": total_nr_genes,
"NumberOfERCCReads": total_ercc_reads,
"NumberOfChromReads": total_chromosomal_reads,
"NumberOfMTReads": mitochondrial_reads,
}
total_nr_reads_per_chromosome = total_nr_reads_per_chromosome.assign(
pctChrom=percentage_chromosomal_reads,
pctMT=percentage_mitochondrial_reads,
pctERCC=percentage_ercc_reads,
SumReads=total_sum_of_reads,
NumberOfGenes=total_nr_genes,
**cols_to_add
)
total_nr_reads_per_chromosome.reset_index(names="WellBC")\\
.to_csv(par["nrReadsNrGenesPerChromPool"], sep="\\t",
header=True, index=False,
columns=("WellBC",) + tuple(chromosome_names) + \\
("SumReads", "pctMT", "pctERCC", "pctChrom", "NumberOfGenes"))
header=True, index=False, float_format="%g",
columns=("WellBC",) + tuple(chromosome_names) + tuple(cols_to_add.keys())
)
VIASHMAIN
python -B "\$tempscript" &
wait "\$!"

86
target/executable/stats/generate_pool_statistics/nextflow_labels.config
View File

@@ -1,26 +1,88 @@
executor {
$k8s {
submitRateLimit = '10sec'
pollInterval = '1 sec'
}
}
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
container = 'nextflow/bash:latest'
// default resources
memory = { 8.Gb * task.attempt }
cpus = 8
maxForks = 36
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
maxMemory = 192.GB
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 8 }
withLabel: midcpu { cpus = 16 }
withLabel: highcpu { cpus = 32 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
}
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
local {
// This config is for local processing.
process {
maxMemory = 25.GB
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 6 }
withLabel: highcpu { cpus = 12 }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
}
}
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare

13
target/executable/stats/generate_well_statistics/.config.vsh.yaml
View File

@@ -225,15 +225,18 @@ build_info:
engine: "docker|native"
output: "target/executable/stats/generate_well_statistics"
executable: "target/executable/stats/generate_well_statistics/generate_well_statistics"
viash_version: "0.9.0-RC7"
git_commit: "cf9797232db1306bfd5696287928cababe317d99"
git_remote: "https://x-access-token:ghs_KjB7pWu8DQM3iFulLu7RI06qnt5K8S1A0eaE@github.com/viash-hub/htrnaseq"
viash_version: "0.9.0"
git_commit: "65dd41d8b1b4a307735c72320c96c0880c75f17f"
git_remote: "https://x-access-token:ghs_McZDF0yobnnHmOEb2Q4JaaB3pzr9mz1VbIOs@github.com/viash-hub/htrnaseq"
package_config:
name: "htrnaseq"
version: "main"
description: "High-throughput pipeline [WIP]\n"
info: null
viash_version: "0.9.0-RC7"
info:
test_resources:
- path: "gs://viash-hub-test-data/htrnaseq/v1/"
dest: "resources_test"
viash_version: "0.9.0"
source: "src"
target: "target"
config_mods:

10
target/executable/stats/generate_well_statistics/generate_well_statistics
View File

@@ -2,9 +2,9 @@
# generate_well_statistics main
#
# This wrapper script is auto-generated by viash 0.9.0-RC7 and is thus a
# derivative work thereof. This software comes with ABSOLUTELY NO WARRANTY from
# Data Intuitive.
# This wrapper script is auto-generated by viash 0.9.0 and is thus a derivative
# work thereof. This software comes with ABSOLUTELY NO WARRANTY from Data
# Intuitive.
#
# The component may contain files which fall under a different license. The
# authors of this component should specify the license in the header of such
@@ -511,9 +511,9 @@ RUN pip install --upgrade pip && \
pip install --upgrade --no-cache-dir "pysam" "pandas"
LABEL org.opencontainers.image.description="Companion container for running component stats generate_well_statistics"
LABEL org.opencontainers.image.created="2024-09-17T08:52:48Z"
LABEL org.opencontainers.image.created="2024-11-05T15:39:43Z"
LABEL org.opencontainers.image.source="https://github.com/viash-hub/htrnaseq"
LABEL org.opencontainers.image.revision="cf9797232db1306bfd5696287928cababe317d99"
LABEL org.opencontainers.image.revision="65dd41d8b1b4a307735c72320c96c0880c75f17f"
LABEL org.opencontainers.image.version="main"
VIASHDOCKER

86
target/executable/stats/generate_well_statistics/nextflow_labels.config
View File

@@ -1,26 +1,88 @@
executor {
$k8s {
submitRateLimit = '10sec'
pollInterval = '1 sec'
}
}
process {
// Default resources for components that hardly do any processing
memory = { 2.GB * task.attempt }
cpus = 1
container = 'nextflow/bash:latest'
// default resources
memory = { 8.Gb * task.attempt }
cpus = 8
maxForks = 36
// Retry for exit codes that have something to do with memory issues
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
maxRetries = 3
maxMemory = null
maxMemory = 192.GB
// Resource labels
withLabel: singlecpu { cpus = 1 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 10 }
withLabel: highcpu { cpus = 20 }
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 8 }
withLabel: midcpu { cpus = 16 }
withLabel: highcpu { cpus = 32 }
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
}
profiles {
// detect tempdir
tempDir = java.nio.file.Paths.get(
System.getenv('NXF_TEMP') ?:
System.getenv('VIASH_TEMP') ?:
System.getenv('TEMPDIR') ?:
System.getenv('TMPDIR') ?:
'/tmp'
).toAbsolutePath()
mount_temp {
docker.temp = tempDir
podman.temp = tempDir
charliecloud.temp = tempDir
}
no_publish {
process {
withName: '.*' {
publishDir = [
enabled: false
]
}
}
}
docker {
docker.fixOwnership = true
docker.enabled = true
// docker.userEmulation = true
singularity.enabled = false
podman.enabled = false
shifter.enabled = false
charliecloud.enabled = false
}
local {
// This config is for local processing.
process {
maxMemory = 25.GB
withLabel: verylowcpu { cpus = 2 }
withLabel: lowcpu { cpus = 4 }
withLabel: midcpu { cpus = 6 }
withLabel: highcpu { cpus = 12 }
withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
}
}
}
def get_memory(to_compare) {
if (!process.containsKey("maxMemory") || !process.maxMemory) {
return to_compare