Build branch fix-integration-tests with version dev (2dbe3b72)
Build pipeline: vsh-ci-dev-k8tz4
Source commit: 2dbe3b7231
Source message: Fix pointers to test resources
This commit is contained in:
@@ -0,0 +1,304 @@
|
||||
name: "build_bdrhap_reference"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Robrecht Cannoodt"
|
||||
roles:
|
||||
- "author"
|
||||
- "maintainer"
|
||||
info:
|
||||
role: "Core Team Member"
|
||||
links:
|
||||
email: "robrecht@data-intuitive.com"
|
||||
github: "rcannood"
|
||||
orcid: "0000-0003-3641-729X"
|
||||
linkedin: "robrechtcannoodt"
|
||||
organizations:
|
||||
- name: "Data Intuitive"
|
||||
href: "https://www.data-intuitive.com"
|
||||
role: "Data Science Engineer"
|
||||
- name: "Open Problems"
|
||||
href: "https://openproblems.bio"
|
||||
role: "Core Member"
|
||||
- name: "Weiwei Schultz"
|
||||
roles:
|
||||
- "contributor"
|
||||
info:
|
||||
role: "Contributor"
|
||||
organizations:
|
||||
- name: "Janssen R&D US"
|
||||
role: "Associate Director Data Sciences"
|
||||
argument_groups:
|
||||
- name: "Inputs"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--genome_fasta"
|
||||
description: "Reference genome file in FASTA or FASTA.GZ format. The BD Rhapsody\
|
||||
\ Sequencing Analysis Pipeline uses GRCh38 for Human and GRCm39 for Mouse."
|
||||
info:
|
||||
config_key: "Genome_fasta"
|
||||
example:
|
||||
- "genome_sequence.fa.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--gtf"
|
||||
description: "File path to the transcript annotation files in GTF or GTF.GZ format.\
|
||||
\ The Sequence Analysis Pipeline requires the 'gene_name' or \n'gene_id' attribute\
|
||||
\ to be set on each gene and exon feature. Gene and exon feature lines must\
|
||||
\ have the same attribute, and exons\nmust have a corresponding gene with the\
|
||||
\ same value. For TCR/BCR assays, the TCR or BCR gene segments must have the\
|
||||
\ 'gene_type' or\n'gene_biotype' attribute set, and the value should begin with\
|
||||
\ 'TR' or 'IG', respectively.\n"
|
||||
info:
|
||||
config_key: "Gtf"
|
||||
example:
|
||||
- "transcriptome_annotation.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--extra_sequences"
|
||||
description: "File path to additional sequences in FASTA format to use when building\
|
||||
\ the STAR index. (e.g. transgenes or CRISPR guide barcodes).\nGTF lines for\
|
||||
\ these sequences will be automatically generated and combined with the main\
|
||||
\ GTF.\n"
|
||||
info:
|
||||
config_key: "Extra_sequences"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- name: "Outputs"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--reference_archive"
|
||||
description: "A Compressed archive containing the Reference Genome Index and annotation\
|
||||
\ GTF files. This archive is meant to be used as an\ninput in the BD Rhapsody\
|
||||
\ Sequencing Analysis Pipeline.\n"
|
||||
info: null
|
||||
example:
|
||||
- "reference.tar.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- name: "Arguments"
|
||||
arguments:
|
||||
- type: "string"
|
||||
name: "--mitochondrial_contigs"
|
||||
description: "Names of the Mitochondrial contigs in the provided Reference Genome.\
|
||||
\ Fragments originating from contigs other than these are\nidentified as 'nuclear\
|
||||
\ fragments' in the ATACseq analysis pipeline.\n"
|
||||
info:
|
||||
config_key: "Mitochondrial_contigs"
|
||||
default:
|
||||
- "chrM"
|
||||
- "chrMT"
|
||||
- "M"
|
||||
- "MT"
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- type: "boolean_true"
|
||||
name: "--filtering_off"
|
||||
description: "By default the input Transcript Annotation files are filtered based\
|
||||
\ on the gene_type/gene_biotype attribute. Only features \nhaving the following\
|
||||
\ attribute values are kept:\n\n - protein_coding\n - lncRNA \n - IG_LV_gene\n\
|
||||
\ - IG_V_gene\n - IG_V_pseudogene\n - IG_D_gene\n - IG_J_gene\n - IG_J_pseudogene\n\
|
||||
\ - IG_C_gene\n - IG_C_pseudogene\n - TR_V_gene\n - TR_V_pseudogene\n -\
|
||||
\ TR_D_gene\n - TR_J_gene\n - TR_J_pseudogene\n - TR_C_gene\n\n If you have\
|
||||
\ already pre-filtered the input Annotation files and/or wish to turn-off the\
|
||||
\ filtering, please set this option to True.\n"
|
||||
info:
|
||||
config_key: "Filtering_off"
|
||||
direction: "input"
|
||||
- type: "boolean_true"
|
||||
name: "--wta_only_index"
|
||||
description: "Build a WTA only index, otherwise builds a WTA + ATAC index."
|
||||
info:
|
||||
config_key: "Wta_Only"
|
||||
direction: "input"
|
||||
- type: "string"
|
||||
name: "--extra_star_params"
|
||||
description: "Additional parameters to pass to STAR when building the genome index.\
|
||||
\ Specify exactly like how you would on the command line."
|
||||
info:
|
||||
config_key: "Extra_STAR_params"
|
||||
example:
|
||||
- "--limitGenomeGenerateRAM 48000 --genomeSAindexNbases 11"
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "python_script"
|
||||
path: "script.py"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "make_rhap_reference_2.2.1_nodocker.cwl"
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "The Reference Files Generator creates an archive containing Genome Index\n\
|
||||
and Transcriptome annotation files needed for the BD Rhapsody Sequencing\nAnalysis\
|
||||
\ Pipeline. The app takes as input one or more FASTA and GTF files\nand produces\
|
||||
\ a compressed archive in the form of a tar.gz file. The \narchive contains:\n \
|
||||
\ \n- STAR index\n- Filtered GTF file\n"
|
||||
test_resources:
|
||||
- type: "bash_script"
|
||||
path: "run_test.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "reference.fa.gz"
|
||||
- type: "file"
|
||||
path: "reference.gtf.gz"
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "highmem"
|
||||
- "highcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "bdgenomics/rhapsody:2.2.1"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "apt"
|
||||
packages:
|
||||
- "procps"
|
||||
- "seqkit"
|
||||
interactive: false
|
||||
- type: "python"
|
||||
user: false
|
||||
packages:
|
||||
- "cwlref-runner"
|
||||
- "cwl-runner"
|
||||
upgrade: true
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/build_bdrhap_reference/config.vsh.yaml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/build_bdrhap_reference"
|
||||
executable: "target/executable/reference/build_bdrhap_reference/build_bdrhap_reference"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
1509
target/executable/reference/build_bdrhap_reference/build_bdrhap_reference
Executable file
1509
target/executable/reference/build_bdrhap_reference/build_bdrhap_reference
Executable file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,115 @@
|
||||
requirements:
|
||||
InlineJavascriptRequirement: {}
|
||||
class: CommandLineTool
|
||||
label: Reference Files Generator for BD Rhapsodyâ„¢ Sequencing Analysis Pipeline
|
||||
cwlVersion: v1.2
|
||||
doc: >-
|
||||
The Reference Files Generator creates an archive containing Genome Index and Transcriptome annotation files needed for the BD Rhapsodyâ„¢ Sequencing Analysis Pipeline. The app takes as input one or more FASTA and GTF files and produces a compressed archive in the form of a tar.gz file. The archive contains:\n - STAR index\n - Filtered GTF file
|
||||
|
||||
|
||||
baseCommand: run_reference_generator.sh
|
||||
inputs:
|
||||
Genome_fasta:
|
||||
type: File[]
|
||||
label: Reference Genome
|
||||
doc: |-
|
||||
Reference genome file in FASTA format. The BD Rhapsodyâ„¢ Sequencing Analysis Pipeline uses GRCh38 for Human and GRCm39 for Mouse.
|
||||
inputBinding:
|
||||
prefix: --reference-genome
|
||||
shellQuote: false
|
||||
Gtf:
|
||||
type: File[]
|
||||
label: Transcript Annotations
|
||||
doc: |-
|
||||
Transcript annotation files in GTF format. The BD Rhapsodyâ„¢ Sequencing Analysis Pipeline uses Gencode v42 for Human and M31 for Mouse.
|
||||
inputBinding:
|
||||
prefix: --gtf
|
||||
shellQuote: false
|
||||
Extra_sequences:
|
||||
type: File[]?
|
||||
label: Extra Sequences
|
||||
doc: |-
|
||||
Additional sequences in FASTA format to use when building the STAR index. (E.g. phiX genome)
|
||||
inputBinding:
|
||||
prefix: --extra-sequences
|
||||
shellQuote: false
|
||||
Mitochondrial_Contigs:
|
||||
type: string[]?
|
||||
default: ["chrM", "chrMT", "M", "MT"]
|
||||
label: Mitochondrial Contig Names
|
||||
doc: |-
|
||||
Names of the Mitochondrial contigs in the provided Reference Genome. Fragments originating from contigs other than these are identified as 'nuclear fragments' in the ATACseq analysis pipeline.
|
||||
inputBinding:
|
||||
prefix: --mitochondrial-contigs
|
||||
shellQuote: false
|
||||
Filtering_off:
|
||||
type: boolean?
|
||||
label: Turn off filtering
|
||||
doc: |-
|
||||
By default the input Transcript Annotation files are filtered based on the gene_type/gene_biotype attribute. Only features having the following attribute values are are kept:
|
||||
- protein_coding
|
||||
- lncRNA (lincRNA and antisense for Gencode < v31/M22/Ensembl97)
|
||||
- IG_LV_gene
|
||||
- IG_V_gene
|
||||
- IG_V_pseudogene
|
||||
- IG_D_gene
|
||||
- IG_J_gene
|
||||
- IG_J_pseudogene
|
||||
- IG_C_gene
|
||||
- IG_C_pseudogene
|
||||
- TR_V_gene
|
||||
- TR_V_pseudogene
|
||||
- TR_D_gene
|
||||
- TR_J_gene
|
||||
- TR_J_pseudogene
|
||||
- TR_C_gene
|
||||
If you have already pre-filtered the input Annotation files and/or wish to turn-off the filtering, please set this option to True.
|
||||
inputBinding:
|
||||
prefix: --filtering-off
|
||||
shellQuote: false
|
||||
WTA_Only:
|
||||
type: boolean?
|
||||
label: WTA only index
|
||||
doc: Build a WTA only index, otherwise builds a WTA + ATAC index.
|
||||
inputBinding:
|
||||
prefix: --wta-only-index
|
||||
shellQuote: false
|
||||
Archive_prefix:
|
||||
type: string?
|
||||
label: Archive Prefix
|
||||
doc: |-
|
||||
A prefix for naming the compressed archive file containing the Reference genome index and annotation files. The default value is constructed based on the input Reference files.
|
||||
inputBinding:
|
||||
prefix: --archive-prefix
|
||||
shellQuote: false
|
||||
Extra_STAR_params:
|
||||
type: string?
|
||||
label: Extra STAR Params
|
||||
doc: |-
|
||||
Additional parameters to pass to STAR when building the genome index. Specify exactly like how you would on the command line.
|
||||
Example:
|
||||
--limitGenomeGenerateRAM 48000 --genomeSAindexNbases 11
|
||||
inputBinding:
|
||||
prefix: --extra-star-params
|
||||
shellQuote: true
|
||||
|
||||
Maximum_threads:
|
||||
type: int?
|
||||
label: Maximum Number of Threads
|
||||
doc: |-
|
||||
The maximum number of threads to use in the pipeline. By default, all available cores are used.
|
||||
inputBinding:
|
||||
prefix: --maximum-threads
|
||||
shellQuote: false
|
||||
|
||||
outputs:
|
||||
|
||||
Archive:
|
||||
type: File
|
||||
doc: |-
|
||||
A Compressed archive containing the Reference Genome Index and annotation GTF files. This archive is meant to be used as an input in the BD Rhapsodyâ„¢ Sequencing Analysis Pipeline.
|
||||
id: Reference_Archive
|
||||
label: Reference Files Archive
|
||||
outputBinding:
|
||||
glob: '*.tar.gz'
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
name: "build_cellranger_arc_reference"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Vladimir Shitov"
|
||||
roles:
|
||||
- "author"
|
||||
info:
|
||||
role: "Contributor"
|
||||
links:
|
||||
email: "vladimir.shitov@helmholtz-muenchen.de"
|
||||
github: "vladimirshitov"
|
||||
orcid: "0000-0002-1960-8812"
|
||||
linkedin: "vladimir-shitov-9a659513b"
|
||||
organizations:
|
||||
- name: "Helmholtz Munich"
|
||||
href: "https://www.helmholtz-munich.de"
|
||||
role: "PhD Candidate"
|
||||
argument_groups:
|
||||
- name: "Arguments"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--genome_fasta"
|
||||
description: "Reference genome fasta."
|
||||
info: null
|
||||
example:
|
||||
- "genome_sequence.fa.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--annotation_gtf"
|
||||
description: "Reference annotation."
|
||||
info: null
|
||||
example:
|
||||
- "annotation.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--motifs_file"
|
||||
description: "Transcription factor motifs in JASPAR format. See https://support.10xgenomics.com/single-cell-multiome-atac-gex/software/pipelines/latest/advanced/references"
|
||||
info: null
|
||||
example:
|
||||
- "JASPAR2024_CORE_non-redundant_pfms_jaspar.txt.modified"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--non_nuclear_contigs"
|
||||
description: "Name(s) of contig(s) that do not have any chromatin structure, for\
|
||||
\ example, mitochondria or plastids. These contigs are excluded from peak calling\
|
||||
\ since the entire contig will be \"open\" due to a lack of chromatin structure.\
|
||||
\ Leave empty if there are no such contigs."
|
||||
info: null
|
||||
example:
|
||||
- "chrM"
|
||||
default:
|
||||
- "chrM"
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output"
|
||||
description: "Output folder"
|
||||
info: null
|
||||
example:
|
||||
- "cellranger_reference"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--genome"
|
||||
description: "Name of the genome. This will be the name of the intermediate output\
|
||||
\ folder"
|
||||
info: null
|
||||
example:
|
||||
- "GRCh38"
|
||||
default:
|
||||
- "output"
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--organism"
|
||||
description: "Name of the organism. This is displayed in the web summary but is\
|
||||
\ otherwise not used in the analysis."
|
||||
info: null
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--subset_regex"
|
||||
description: "Will subset the reference chromosomes using the given regex."
|
||||
info: null
|
||||
example:
|
||||
- "(ERCC-00002|chr1)"
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "bash_script"
|
||||
path: "script.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Build a Cell Ranger-arc and -atac compatible reference folder from user-supplied\
|
||||
\ genome FASTA and gene GTF files. Creates a new folder named after the genome."
|
||||
test_resources:
|
||||
- type: "bash_script"
|
||||
path: "run_test.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "reference_gencodev41_chr1"
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "highmem"
|
||||
- "highcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "ghcr.io/data-intuitive/cellranger_arc:2.0"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "docker"
|
||||
run:
|
||||
- "DEBIAN_FRONTEND=noninteractive apt update && \\\napt upgrade -y && apt install\
|
||||
\ -y procps pigz && rm -rf /var/lib/apt/lists/*\n"
|
||||
test_setup:
|
||||
- type: "docker"
|
||||
- type: "apt"
|
||||
packages:
|
||||
- "git"
|
||||
- "wget"
|
||||
interactive: false
|
||||
- type: "docker"
|
||||
run:
|
||||
- "TARGETARCH=\"${TARGETARCH:-$(dpkg --print-architecture)}\" && \\\nTARGETOS=\"\
|
||||
${TARGETOS:-linux}\" && \\\nPATH=\"${PATH}:/usr/local/go/bin\" && \\\nwget https://go.dev/dl/go1.21.4.$TARGETOS-$TARGETARCH.tar.gz\
|
||||
\ && tar -C /usr/local/ -xzf go1.21.4.$TARGETOS-$TARGETARCH.tar.gz && \\\nrm\
|
||||
\ go1.21.4.$TARGETOS-$TARGETARCH.tar.gz && \\\ngit clone --branch v2.5.0 https://github.com/shenwei356/seqkit.git\
|
||||
\ && \\\ncd seqkit/seqkit/ && go build && cp seqkit /usr/bin/ && cd ../../ &&\
|
||||
\ rm -rf seqkit && rm -r /usr/local/go\n"
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/build_cellranger_arc_reference/config.vsh.yaml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/build_cellranger_arc_reference"
|
||||
executable: "target/executable/reference/build_cellranger_arc_reference/build_cellranger_arc_reference"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,244 @@
|
||||
name: "build_cellranger_reference"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Angela Oliveira Pisco"
|
||||
roles:
|
||||
- "author"
|
||||
info:
|
||||
role: "Contributor"
|
||||
links:
|
||||
github: "aopisco"
|
||||
orcid: "0000-0003-0142-2355"
|
||||
linkedin: "aopisco"
|
||||
organizations:
|
||||
- name: "Insitro"
|
||||
href: "https://insitro.com"
|
||||
role: "Director of Computational Biology"
|
||||
- name: "Open Problems"
|
||||
href: "https://openproblems.bio"
|
||||
role: "Core Member"
|
||||
- name: "Robrecht Cannoodt"
|
||||
roles:
|
||||
- "author"
|
||||
- "maintainer"
|
||||
info:
|
||||
role: "Core Team Member"
|
||||
links:
|
||||
email: "robrecht@data-intuitive.com"
|
||||
github: "rcannood"
|
||||
orcid: "0000-0003-3641-729X"
|
||||
linkedin: "robrechtcannoodt"
|
||||
organizations:
|
||||
- name: "Data Intuitive"
|
||||
href: "https://www.data-intuitive.com"
|
||||
role: "Data Science Engineer"
|
||||
- name: "Open Problems"
|
||||
href: "https://openproblems.bio"
|
||||
role: "Core Member"
|
||||
argument_groups:
|
||||
- name: "Arguments"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--genome_fasta"
|
||||
description: "Reference genome fasta."
|
||||
info: null
|
||||
example:
|
||||
- "genome_sequence.fa.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--transcriptome_gtf"
|
||||
description: "Reference transcriptome annotation."
|
||||
info: null
|
||||
example:
|
||||
- "transcriptome_annotation.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--reference_version"
|
||||
description: "Optional reference version string to include with reference"
|
||||
info: null
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output"
|
||||
description: "Output folder"
|
||||
info: null
|
||||
example:
|
||||
- "cellranger_reference"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "bash_script"
|
||||
path: "script.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Build a Cell Ranger-compatible reference folder from user-supplied genome\
|
||||
\ FASTA and gene GTF files. Creates a new folder named after the genome."
|
||||
test_resources:
|
||||
- type: "bash_script"
|
||||
path: "run_test.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "reference_gencodev41_chr1"
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "highmem"
|
||||
- "highcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "ghcr.io/data-intuitive/cellranger:8.0"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "docker"
|
||||
run:
|
||||
- "DEBIAN_FRONTEND=noninteractive apt update && \\\napt upgrade -y && apt install\
|
||||
\ -y procps pigz && rm -rf /var/lib/apt/lists/*\n"
|
||||
test_setup:
|
||||
- type: "apt"
|
||||
packages:
|
||||
- "git"
|
||||
- "wget"
|
||||
interactive: false
|
||||
- type: "docker"
|
||||
run:
|
||||
- "TARGETARCH=\"${TARGETARCH:-$(dpkg --print-architecture)}\" && \\\nTARGETOS=\"\
|
||||
${TARGETOS:-linux}\" && \\\nPATH=\"${PATH}:/usr/local/go/bin\" && \\\nwget https://go.dev/dl/go1.21.4.$TARGETOS-$TARGETARCH.tar.gz\
|
||||
\ && tar -C /usr/local/ -xzf go1.21.4.$TARGETOS-$TARGETARCH.tar.gz && \\\nrm\
|
||||
\ go1.21.4.$TARGETOS-$TARGETARCH.tar.gz && \\\ngit clone --branch v2.5.0 https://github.com/shenwei356/seqkit.git\
|
||||
\ && \\\ncd seqkit/seqkit/ && go build && cp seqkit /usr/bin/ && cd ../../ &&\
|
||||
\ rm -rf seqkit && rm -r /usr/local/go\n"
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/build_cellranger_reference/config.vsh.yaml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/build_cellranger_reference"
|
||||
executable: "target/executable/reference/build_cellranger_reference/build_cellranger_reference"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
1160
target/executable/reference/build_cellranger_reference/build_cellranger_reference
Executable file
1160
target/executable/reference/build_cellranger_reference/build_cellranger_reference
Executable file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,238 @@
|
||||
name: "build_star_reference"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Dries Schaumont"
|
||||
roles:
|
||||
- "author"
|
||||
info:
|
||||
role: "Core Team Member"
|
||||
links:
|
||||
email: "dries@data-intuitive.com"
|
||||
github: "DriesSchaumont"
|
||||
orcid: "0000-0002-4389-0440"
|
||||
linkedin: "dries-schaumont"
|
||||
organizations:
|
||||
- name: "Data Intuitive"
|
||||
href: "https://www.data-intuitive.com"
|
||||
role: "Data Scientist"
|
||||
argument_groups:
|
||||
- name: "Input/Output"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--genome_fasta"
|
||||
alternatives:
|
||||
- "--genomeFastaFiles"
|
||||
description: "The fasta files to be included in the reference. Corresponds to\
|
||||
\ the --genomeFastaFiles argument in the STAR command."
|
||||
info: null
|
||||
example:
|
||||
- "chr1.fasta"
|
||||
- "chr2.fasta"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--transcriptome_gtf"
|
||||
alternatives:
|
||||
- "--sjdbGTFfile"
|
||||
description: "Specifies the path to the file with annotated transcripts in the\
|
||||
\ standard GTF\nformat. STAR will extract splice junctions from this file and\
|
||||
\ use them to greatly improve\naccuracy of the mapping. Corresponds to the --sjdbGTFfile\
|
||||
\ argument in the STAR command.\n"
|
||||
info: null
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output"
|
||||
alternatives:
|
||||
- "--genomeDir"
|
||||
description: "Path to output directory. Corresponds to the --genomeDir argument\
|
||||
\ in the STAR command."
|
||||
info: null
|
||||
example:
|
||||
- "/path/to/foo"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- name: "Genome indexing arguments"
|
||||
arguments:
|
||||
- type: "integer"
|
||||
name: "--genomeSAindexNbases"
|
||||
description: "Length (bases) of the SA pre-indexing string. Typically between\
|
||||
\ 10 and 15.\nLonger strings will use much more memory, but allow faster searches.\
|
||||
\ For small\ngenomes, the parameter {genomeSAindexNbases must be scaled down\
|
||||
\ to\nmin(14, log2(GenomeLength)/2 - 1).\n"
|
||||
info: null
|
||||
default:
|
||||
- 14
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "python_script"
|
||||
path: "script.py"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Create a reference for STAR from a set of fasta files."
|
||||
test_resources:
|
||||
- type: "bash_script"
|
||||
path: "test.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "cellranger_tiny_fastq"
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "highmem"
|
||||
- "highcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "python:3.10-slim"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "apt"
|
||||
packages:
|
||||
- "procps"
|
||||
interactive: false
|
||||
- type: "docker"
|
||||
env:
|
||||
- "STAR_VERSION 2.7.10b"
|
||||
- "PACKAGES gcc g++ make wget zlib1g-dev unzip"
|
||||
- type: "docker"
|
||||
run:
|
||||
- "apt-get update && \\\n apt-get install -y --no-install-recommends ${PACKAGES}\
|
||||
\ && \\\n cd /tmp && \\\n wget --no-check-certificate https://github.com/alexdobin/STAR/archive/refs/tags/${STAR_VERSION}.zip\
|
||||
\ && \\\n unzip ${STAR_VERSION}.zip && \\\n cd STAR-${STAR_VERSION}/source\
|
||||
\ && \\\n make STARstatic CXXFLAGS_SIMD=-std=c++11 && \\\n cp STAR /usr/local/bin\
|
||||
\ && \\\n cd / && \\\n rm -rf /tmp/STAR-${STAR_VERSION} /tmp/${STAR_VERSION}.zip\
|
||||
\ && \\\n apt-get --purge autoremove -y ${PACKAGES} && \\\n apt-get clean\n"
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/build_star_reference/config.vsh.yml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/build_star_reference"
|
||||
executable: "target/executable/reference/build_star_reference/build_star_reference"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
1355
target/executable/reference/build_star_reference/build_star_reference
Executable file
1355
target/executable/reference/build_star_reference/build_star_reference
Executable file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
214
target/executable/reference/cellranger_mkgtf/.config.vsh.yaml
Normal file
214
target/executable/reference/cellranger_mkgtf/.config.vsh.yaml
Normal file
@@ -0,0 +1,214 @@
|
||||
name: "cellranger_mkgtf"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Jakub Majercik"
|
||||
roles:
|
||||
- "author"
|
||||
info:
|
||||
role: "Contributor"
|
||||
links:
|
||||
email: "jakub@data-intuitive.com"
|
||||
github: "jakubmajercik"
|
||||
linkedin: "jakubmajercik"
|
||||
organizations:
|
||||
- name: "Data Intuitive"
|
||||
href: "https://www.data-intuitive.com"
|
||||
role: "Bioinformatics Engineer"
|
||||
argument_groups:
|
||||
- name: "Arguments"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--input_gtf"
|
||||
description: "Reference GTF annotation."
|
||||
info: null
|
||||
example:
|
||||
- "transcriptome_annotation.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output_gtf"
|
||||
description: "Output GTF file."
|
||||
info: null
|
||||
example:
|
||||
- "output.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--attribute"
|
||||
description: "Key-value pair in attributes field to be kept in the GTF file of\
|
||||
\ the format attribute:attribute_value."
|
||||
info: null
|
||||
example:
|
||||
- "gene_type:transcribed_unprocessed_pseudogene"
|
||||
- "gene_type:miRNA"
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: true
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "bash_script"
|
||||
path: "script.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Make a GTF file - filter by a specific attribute."
|
||||
test_resources:
|
||||
- type: "python_script"
|
||||
path: "test.py"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "reference_gencodev41_chr1"
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "midmem"
|
||||
- "lowcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "ghcr.io/data-intuitive/cellranger:8.0"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "docker"
|
||||
run:
|
||||
- "DEBIAN_FRONTEND=noninteractive apt update && \\\napt upgrade -y && apt install\
|
||||
\ -y pigz procps && rm -rf /var/lib/apt/lists/*\n"
|
||||
test_setup:
|
||||
- type: "docker"
|
||||
copy:
|
||||
- "openpipelinetestutils /opt/openpipelinetestutils"
|
||||
- type: "python"
|
||||
user: false
|
||||
packages:
|
||||
- "/opt/openpipelinetestutils"
|
||||
upgrade: true
|
||||
- type: "python"
|
||||
user: false
|
||||
packages:
|
||||
- "viashpy==0.8.0"
|
||||
upgrade: true
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/cellranger_mkgtf/config.vsh.yaml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/cellranger_mkgtf"
|
||||
executable: "target/executable/reference/cellranger_mkgtf/cellranger_mkgtf"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
1143
target/executable/reference/cellranger_mkgtf/cellranger_mkgtf
Executable file
1143
target/executable/reference/cellranger_mkgtf/cellranger_mkgtf
Executable file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
261
target/executable/reference/make_reference/.config.vsh.yaml
Normal file
261
target/executable/reference/make_reference/.config.vsh.yaml
Normal file
@@ -0,0 +1,261 @@
|
||||
name: "make_reference"
|
||||
namespace: "reference"
|
||||
version: "dev"
|
||||
authors:
|
||||
- name: "Angela Oliveira Pisco"
|
||||
roles:
|
||||
- "author"
|
||||
info:
|
||||
role: "Contributor"
|
||||
links:
|
||||
github: "aopisco"
|
||||
orcid: "0000-0003-0142-2355"
|
||||
linkedin: "aopisco"
|
||||
organizations:
|
||||
- name: "Insitro"
|
||||
href: "https://insitro.com"
|
||||
role: "Director of Computational Biology"
|
||||
- name: "Open Problems"
|
||||
href: "https://openproblems.bio"
|
||||
role: "Core Member"
|
||||
- name: "Robrecht Cannoodt"
|
||||
roles:
|
||||
- "author"
|
||||
- "maintainer"
|
||||
info:
|
||||
role: "Core Team Member"
|
||||
links:
|
||||
email: "robrecht@data-intuitive.com"
|
||||
github: "rcannood"
|
||||
orcid: "0000-0003-3641-729X"
|
||||
linkedin: "robrechtcannoodt"
|
||||
organizations:
|
||||
- name: "Data Intuitive"
|
||||
href: "https://www.data-intuitive.com"
|
||||
role: "Data Science Engineer"
|
||||
- name: "Open Problems"
|
||||
href: "https://openproblems.bio"
|
||||
role: "Core Member"
|
||||
argument_groups:
|
||||
- name: "Arguments"
|
||||
arguments:
|
||||
- type: "file"
|
||||
name: "--genome_fasta"
|
||||
description: "Reference genome fasta. Example: "
|
||||
info: null
|
||||
example:
|
||||
- "genome_fasta.fa.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--transcriptome_gtf"
|
||||
description: "Reference transcriptome annotation."
|
||||
info: null
|
||||
example:
|
||||
- "transcriptome.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--ercc"
|
||||
description: "ERCC sequence and annotation file."
|
||||
info: null
|
||||
example:
|
||||
- "ercc.zip"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "string"
|
||||
name: "--subset_regex"
|
||||
description: "Will subset the reference chromosomes using the given regex."
|
||||
info: null
|
||||
example:
|
||||
- "(ERCC-00002|chr1)"
|
||||
required: false
|
||||
direction: "input"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output_fasta"
|
||||
description: "Output genome sequence fasta."
|
||||
info: null
|
||||
example:
|
||||
- "genome_sequence.fa.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
- type: "file"
|
||||
name: "--output_gtf"
|
||||
description: "Output transcriptome annotation gtf."
|
||||
info: null
|
||||
example:
|
||||
- "transcriptome_annotation.gtf.gz"
|
||||
must_exist: true
|
||||
create_parent: true
|
||||
required: true
|
||||
direction: "output"
|
||||
multiple: false
|
||||
multiple_sep: ";"
|
||||
resources:
|
||||
- type: "bash_script"
|
||||
path: "script.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "nextflow_labels.config"
|
||||
dest: "nextflow_labels.config"
|
||||
description: "Preprocess and build a transcriptome reference.\n\nExample input files\
|
||||
\ are:\n - `genome_fasta`: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/GRCh38.primary_assembly.genome.fa.gz\n\
|
||||
\ - `transcriptome_gtf`: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_41/gencode.v41.annotation.gtf.gz\n\
|
||||
\ - `ercc`: https://assets.thermofisher.com/TFS-Assets/LSG/manuals/ERCC92.zip\n"
|
||||
test_resources:
|
||||
- type: "bash_script"
|
||||
path: "run_test.sh"
|
||||
is_executable: true
|
||||
- type: "file"
|
||||
path: "openpipelinetestutils"
|
||||
dest: "openpipelinetestutils"
|
||||
info: null
|
||||
status: "enabled"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
runners:
|
||||
- type: "executable"
|
||||
id: "executable"
|
||||
docker_setup_strategy: "ifneedbepullelsecachedbuild"
|
||||
- type: "nextflow"
|
||||
id: "nextflow"
|
||||
directives:
|
||||
label:
|
||||
- "highmem"
|
||||
- "highcpu"
|
||||
tag: "$id"
|
||||
auto:
|
||||
simplifyInput: true
|
||||
simplifyOutput: false
|
||||
transcript: false
|
||||
publish: false
|
||||
config:
|
||||
labels:
|
||||
mem1gb: "memory = 1000000000.B"
|
||||
mem2gb: "memory = 2000000000.B"
|
||||
mem5gb: "memory = 5000000000.B"
|
||||
mem10gb: "memory = 10000000000.B"
|
||||
mem20gb: "memory = 20000000000.B"
|
||||
mem50gb: "memory = 50000000000.B"
|
||||
mem100gb: "memory = 100000000000.B"
|
||||
mem200gb: "memory = 200000000000.B"
|
||||
mem500gb: "memory = 500000000000.B"
|
||||
mem1tb: "memory = 1000000000000.B"
|
||||
mem2tb: "memory = 2000000000000.B"
|
||||
mem5tb: "memory = 5000000000000.B"
|
||||
mem10tb: "memory = 10000000000000.B"
|
||||
mem20tb: "memory = 20000000000000.B"
|
||||
mem50tb: "memory = 50000000000000.B"
|
||||
mem100tb: "memory = 100000000000000.B"
|
||||
mem200tb: "memory = 200000000000000.B"
|
||||
mem500tb: "memory = 500000000000000.B"
|
||||
mem1gib: "memory = 1073741824.B"
|
||||
mem2gib: "memory = 2147483648.B"
|
||||
mem4gib: "memory = 4294967296.B"
|
||||
mem8gib: "memory = 8589934592.B"
|
||||
mem16gib: "memory = 17179869184.B"
|
||||
mem32gib: "memory = 34359738368.B"
|
||||
mem64gib: "memory = 68719476736.B"
|
||||
mem128gib: "memory = 137438953472.B"
|
||||
mem256gib: "memory = 274877906944.B"
|
||||
mem512gib: "memory = 549755813888.B"
|
||||
mem1tib: "memory = 1099511627776.B"
|
||||
mem2tib: "memory = 2199023255552.B"
|
||||
mem4tib: "memory = 4398046511104.B"
|
||||
mem8tib: "memory = 8796093022208.B"
|
||||
mem16tib: "memory = 17592186044416.B"
|
||||
mem32tib: "memory = 35184372088832.B"
|
||||
mem64tib: "memory = 70368744177664.B"
|
||||
mem128tib: "memory = 140737488355328.B"
|
||||
mem256tib: "memory = 281474976710656.B"
|
||||
mem512tib: "memory = 562949953421312.B"
|
||||
cpu1: "cpus = 1"
|
||||
cpu2: "cpus = 2"
|
||||
cpu5: "cpus = 5"
|
||||
cpu10: "cpus = 10"
|
||||
cpu20: "cpus = 20"
|
||||
cpu50: "cpus = 50"
|
||||
cpu100: "cpus = 100"
|
||||
cpu200: "cpus = 200"
|
||||
cpu500: "cpus = 500"
|
||||
cpu1000: "cpus = 1000"
|
||||
script:
|
||||
- "includeConfig(\"nextflow_labels.config\")"
|
||||
debug: false
|
||||
container: "docker"
|
||||
engines:
|
||||
- type: "docker"
|
||||
id: "docker"
|
||||
image: "ubuntu:22.04"
|
||||
target_registry: "images.viash-hub.com"
|
||||
target_tag: "dev"
|
||||
namespace_separator: "/"
|
||||
setup:
|
||||
- type: "apt"
|
||||
packages:
|
||||
- "pigz"
|
||||
- "seqkit"
|
||||
- "curl"
|
||||
- "wget"
|
||||
- "unzip"
|
||||
- "file"
|
||||
interactive: false
|
||||
entrypoint: []
|
||||
cmd: null
|
||||
- type: "native"
|
||||
id: "native"
|
||||
build_info:
|
||||
config: "src/reference/make_reference/config.vsh.yaml"
|
||||
runner: "executable"
|
||||
engine: "docker|native"
|
||||
output: "target/executable/reference/make_reference"
|
||||
executable: "target/executable/reference/make_reference/make_reference"
|
||||
viash_version: "0.9.0"
|
||||
git_commit: "2dbe3b7231f9abb4baa628e76e8abc686e627087"
|
||||
git_remote: "https://x-access-token:ghs_NVsRTpmVPn6SfFQ131njHQOgn6tt7b1bmmJj@github.com/openpipelines-bio/openpipeline"
|
||||
git_tag: "0.2.0-1926-g2dbe3b72"
|
||||
package_config:
|
||||
name: "openpipeline"
|
||||
version: "dev"
|
||||
info:
|
||||
test_resources:
|
||||
- type: "s3"
|
||||
path: "s3://openpipelines-data"
|
||||
dest: "resources_test"
|
||||
viash_version: "0.9.0"
|
||||
source: "src"
|
||||
target: "target"
|
||||
config_mods:
|
||||
- ".test_resources += {path: '/src/base/openpipelinetestutils', dest: 'openpipelinetestutils'}\n\
|
||||
.resources += {path: '/src/workflows/utils/labels.config', dest: 'nextflow_labels.config'}\n\
|
||||
.runners[.type == 'nextflow'].directives.tag := '$id'\n.runners[.type == 'nextflow'].config.script\
|
||||
\ := 'includeConfig(\"nextflow_labels.config\")'"
|
||||
- ".engines += { type: \"native\" }"
|
||||
- ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
|
||||
- ".engines[.type == 'docker'].target_tag := 'dev'"
|
||||
organization: "vsh"
|
||||
links:
|
||||
repository: "https://github.com/openpipelines-bio/openpipeline"
|
||||
docker_registry: "ghcr.io"
|
||||
homepage: "https://openpipelines.bio"
|
||||
documentation: "https://openpipelines.bio/fundamentals"
|
||||
issue_tracker: "https://github.com/openpipelines-bio/openpipeline/issues"
|
||||
1266
target/executable/reference/make_reference/make_reference
Executable file
1266
target/executable/reference/make_reference/make_reference
Executable file
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,42 @@
|
||||
process {
|
||||
// Default resources for components that hardly do any processing
|
||||
memory = { 2.GB * task.attempt }
|
||||
cpus = 1
|
||||
|
||||
// Retry for exit codes that have something to do with memory issues
|
||||
errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
|
||||
maxRetries = 3
|
||||
maxMemory = null
|
||||
|
||||
// Resource labels
|
||||
withLabel: singlecpu { cpus = 1 }
|
||||
withLabel: lowcpu { cpus = 4 }
|
||||
withLabel: midcpu { cpus = 10 }
|
||||
withLabel: highcpu { cpus = 20 }
|
||||
|
||||
withLabel: lowmem { memory = { get_memory( 4.GB * task.attempt ) } }
|
||||
withLabel: midmem { memory = { get_memory( 25.GB * task.attempt ) } }
|
||||
withLabel: highmem { memory = { get_memory( 50.GB * task.attempt ) } }
|
||||
withLabel: veryhighmem { memory = { get_memory( 75.GB * task.attempt ) } }
|
||||
}
|
||||
|
||||
def get_memory(to_compare) {
|
||||
if (!process.containsKey("maxMemory") || !process.maxMemory) {
|
||||
return to_compare
|
||||
}
|
||||
|
||||
try {
|
||||
if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
|
||||
return process.maxMemory
|
||||
}
|
||||
else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
|
||||
return max_memory as nextflow.util.MemoryUnit
|
||||
}
|
||||
else {
|
||||
return to_compare
|
||||
}
|
||||
} catch (all) {
|
||||
println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
|
||||
System.exit(1)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user