Build branch update_craftbox with version update_craftbox (1c55b60)

Build pipeline: viash-hub.htrnaseq.update-craftbox-wm7mk

Source commit: 1c55b60560

Source message: Bump craftbox to v0.3.0

target/executable/parallel_map/.config.vsh.yaml

@@ -0,0 +1,339 @@
+name: "parallel_map"
+version: "update_craftbox"
+authors:
+- name: "Dries Schaumont"
+  roles:
+  - "maintainer"
+  info:
+    links:
+      email: "dries@data-intuitive.com"
+      github: "DriesSchaumont"
+      orcid: "0000-0002-4389-0440"
+      linkedin: "dries-schaumont"
+    organizations:
+    - name: "Data Intuitive"
+      href: "https://www.data-intuitive.com"
+      role: "Data Scientist"
+- name: "Toni Verbeiren"
+  roles:
+  - "author"
+  - "maintainer"
+  info:
+    role: "Core Team Member"
+    links:
+      github: "tverbeiren"
+      linkedin: "verbeiren"
+    organizations:
+    - name: "Data Intuitive"
+      href: "https://www.data-intuitive.com"
+      role: "Data Scientist and CEO"
+argument_groups:
+- name: "Input arguments"
+  arguments:
+  - type: "file"
+    name: "--input_r1"
+    description: "Input FASTQ files for the forward reads. All FASTQ file names must\
+      \ start with the prefix '{well_id}_R1', where\n'well_id' can be found as the\
+      \ sequence identifier in the barcodes FASTA file (see 'barcodesFasta' argument).\n\
+      For each FASTQ file, a matching FASTQ file for the reverse reads must be provided\
+      \ to the 'input_r2' argument,\nmeaning that their 'well_id' prefix must match.\
+      \ The number of items provided for 'input_r1' must be equal\nto the number of\
+      \ items for 'input_r2'.\n"
+    info: null
+    must_exist: true
+    create_parent: true
+    required: true
+    direction: "input"
+    multiple: true
+    multiple_sep: ";"
+  - type: "file"
+    name: "--input_r2"
+    description: "Input FASTQ files for the reverse reads. All FASTQ file names must\
+      \ start with the prefix '{well_id}_R2', where\n'well_id' can be found as the\
+      \ sequence identifier in the barcodes FASTA file (see 'barcodesFasta' argument).\n\
+      For each FASTQ file, a matching FASTQ file for the reverse reads must be provided\
+      \ to the 'input_r1' argument,\nmeaning that their 'well_id' prefix must match.\
+      \ The number of items provided for 'input_r1' must be equal\nto the number of\
+      \ items for 'input_r2'.\n"
+    info: null
+    must_exist: true
+    create_parent: true
+    required: true
+    direction: "input"
+    multiple: true
+    multiple_sep: ";"
+  - type: "file"
+    name: "--genomeDir"
+    description: "Reference genome to match to. Can be generated from genomic FASTA\
+      \ sequences and a genome annotation\nby using STAR with '--runMode genomeGenerate'.\n"
+    info: null
+    must_exist: true
+    create_parent: true
+    required: true
+    direction: "input"
+    multiple: false
+    multiple_sep: ";"
+  - type: "file"
+    name: "--barcodesFasta"
+    description: "FASTA file where each entry specifies a unique barcode sequence\
+      \ present at the start of the forward input reads\n(input_r1). The IDs of each\
+      \ barcode (the start of the FASTA headers up until the first whitespace character)\
+      \ must\nmatch with the start of one input FASTQ pair.\n"
+    info: null
+    must_exist: true
+    create_parent: true
+    required: true
+    direction: "input"
+    multiple: false
+    multiple_sep: ";"
+- name: "Barcode arguments"
+  arguments:
+  - type: "integer"
+    name: "--umiLength"
+    description: "Length of the Unique Molecular Identifiers (UMI). The UMI are expected\
+      \ to be located after the barcodes in the\nforwards reads.\n"
+    info: null
+    required: true
+    direction: "input"
+    multiple: false
+    multiple_sep: ";"
+  - type: "string"
+    name: "--limitBAMsortRAM"
+    info: null
+    default:
+    - "10000000000"
+    required: false
+    direction: "input"
+    multiple: false
+    multiple_sep: ";"
+- name: "Runtime arguments"
+  arguments:
+  - type: "integer"
+    name: "--runThreadN"
+    description: "Number of threads to use for a single STAR execution."
+    info: null
+    default:
+    - 1
+    required: false
+    direction: "input"
+    multiple: false
+    multiple_sep: ";"
+- name: "Output arguments"
+  arguments:
+  - type: "file"
+    name: "--output"
+    description: "A list of output folders which are the result of using STAR to map\
+      \ each input FASTQ pair STAR to the reference genome.\nThe order of the items\
+      \ DO NOT match with the order of the entries in the barcodes FASTA file or the\
+      \ input FASTQ pairs. \n"
+    info: null
+    default:
+    - "./*"
+    must_exist: true
+    create_parent: true
+    required: true
+    direction: "output"
+    multiple: true
+    multiple_sep: ";"
+  - type: "file"
+    name: "--joblog"
+    description: "Where to store the log file listing all the jobs."
+    info: null
+    default:
+    - "execution_log.txt"
+    must_exist: true
+    create_parent: true
+    required: false
+    direction: "output"
+    multiple: false
+    multiple_sep: ";"
+resources:
+- type: "bash_script"
+  path: "script.sh"
+  is_executable: true
+- type: "file"
+  path: "STAR"
+- type: "file"
+  path: "nextflow_labels.config"
+  dest: "nextflow_labels.config"
+- type: "file"
+  path: "_viash.yaml"
+  dest: "_viash.yaml"
+description: "Map wells in batch, using STAR\nSpliced Transcripts Alignment to a Reference\
+  \ (C) Alexander Dobin\nhttps://github.com/alexdobin/STAR\n"
+test_resources:
+- type: "bash_script"
+  path: "test.sh"
+  is_executable: true
+info: null
+status: "enabled"
+scope:
+  image: "public"
+  target: "public"
+requirements:
+  commands:
+  - "ps"
+license: "MIT"
+links:
+  repository: "https://github.com/viash-hub/htrnaseq"
+runners:
+- type: "executable"
+  id: "executable"
+  docker_setup_strategy: "ifneedbepullelsecachedbuild"
+- type: "nextflow"
+  id: "nextflow"
+  directives:
+    tag: "$id"
+  auto:
+    simplifyInput: true
+    simplifyOutput: false
+    transcript: false
+    publish: false
+  config:
+    labels:
+      mem1gb: "memory = 1000000000.B"
+      mem2gb: "memory = 2000000000.B"
+      mem5gb: "memory = 5000000000.B"
+      mem10gb: "memory = 10000000000.B"
+      mem20gb: "memory = 20000000000.B"
+      mem50gb: "memory = 50000000000.B"
+      mem100gb: "memory = 100000000000.B"
+      mem200gb: "memory = 200000000000.B"
+      mem500gb: "memory = 500000000000.B"
+      mem1tb: "memory = 1000000000000.B"
+      mem2tb: "memory = 2000000000000.B"
+      mem5tb: "memory = 5000000000000.B"
+      mem10tb: "memory = 10000000000000.B"
+      mem20tb: "memory = 20000000000000.B"
+      mem50tb: "memory = 50000000000000.B"
+      mem100tb: "memory = 100000000000000.B"
+      mem200tb: "memory = 200000000000000.B"
+      mem500tb: "memory = 500000000000000.B"
+      mem1gib: "memory = 1073741824.B"
+      mem2gib: "memory = 2147483648.B"
+      mem4gib: "memory = 4294967296.B"
+      mem8gib: "memory = 8589934592.B"
+      mem16gib: "memory = 17179869184.B"
+      mem32gib: "memory = 34359738368.B"
+      mem64gib: "memory = 68719476736.B"
+      mem128gib: "memory = 137438953472.B"
+      mem256gib: "memory = 274877906944.B"
+      mem512gib: "memory = 549755813888.B"
+      mem1tib: "memory = 1099511627776.B"
+      mem2tib: "memory = 2199023255552.B"
+      mem4tib: "memory = 4398046511104.B"
+      mem8tib: "memory = 8796093022208.B"
+      mem16tib: "memory = 17592186044416.B"
+      mem32tib: "memory = 35184372088832.B"
+      mem64tib: "memory = 70368744177664.B"
+      mem128tib: "memory = 140737488355328.B"
+      mem256tib: "memory = 281474976710656.B"
+      mem512tib: "memory = 562949953421312.B"
+      cpu1: "cpus = 1"
+      cpu2: "cpus = 2"
+      cpu5: "cpus = 5"
+      cpu10: "cpus = 10"
+      cpu20: "cpus = 20"
+      cpu50: "cpus = 50"
+      cpu100: "cpus = 100"
+      cpu200: "cpus = 200"
+      cpu500: "cpus = 500"
+      cpu1000: "cpus = 1000"
+    script:
+    - "includeConfig(\"nextflow_labels.config\")"
+  debug: false
+  container: "docker"
+engines:
+- type: "docker"
+  id: "docker"
+  image: "debian:stable-slim"
+  target_registry: "images.viash-hub.com"
+  target_tag: "update_craftbox"
+  namespace_separator: "/"
+  setup:
+  - type: "apt"
+    packages:
+    - "procps"
+    - "wget"
+    - "automake"
+    - "make"
+    - "gcc"
+    - "g++"
+    - "zlib1g-dev"
+    - "parallel"
+    - "file"
+    - "seqkit"
+    interactive: false
+  - type: "docker"
+    copy:
+    - "STAR /usr/local/bin/$STAR_BINARY"
+    build_args:
+    - "STAR_V=2.7.6a"
+    env:
+    - "STAR_SOURCE=\"https://github.com/alexdobin/STAR/archive/refs/tags/$STAR_V.tar.gz\""
+    - "STAR_TARGET=\"/app/star-$STAR_V.tar.gz\""
+    - "STAR_INSTALL_DIR=\"/app/STAR-$STAR_V\""
+    - "STAR_BINARY=STAR"
+  entrypoint: []
+  cmd: null
+- type: "native"
+  id: "native"
+build_info:
+  config: "src/parallel_map/config.vsh.yaml"
+  runner: "executable"
+  engine: "docker|native"
+  output: "target/executable/parallel_map"
+  executable: "target/executable/parallel_map/parallel_map"
+  viash_version: "0.9.4"
+  git_commit: "1c55b605604da158bbbc23e79aa1b3d28211790c"
+  git_remote: "https://github.com/viash-hub/htrnaseq"
+  git_tag: "v0.7.2-15-g1c55b60"
+package_config:
+  name: "htrnaseq"
+  version: "update_craftbox"
+  summary: "A workflow for high-throughput RNA-seq data analyses.\n"
+  description: "This workflow is designed to process high-throughput RNA-seq data,\
+    \ where every\nwell of a microarray plate is a sample. A fasta file provided as\
+    \ input\ndefines the mapping between sample barcodes and wells.\n\nThe workflow\
+    \ is built in a modular fashion, where most of the base functionality\nis provided\
+    \ by components from [`biobox`](https://www.viash-hub.com/packages/biobox/latest)\n\
+    supplemented by custom base components and workflow components in this package.\n\
+    \nThe full workflow is split in two major subworkflows that can be run independently:\n\
+    \n* **Well-demultiplexing:** Split the input (plate/pool level) fastq files per\
+    \ well.\n* **Mapping, counting and QC:** Run per-well mapping, counting and generate\
+    \ QC reports.\n\nEach of those can be started individually, or the full workflow\
+    \ can be run in two ways:\n\n1. Run the [main workflow](https://www.viash-hub.com/packages/htrnaseq/v0.3.0/components/workflows/htrnaseq)\
+    \ \ncontaining the main functionality.\n2. Run the [(opinionated) `runner`](https://www.viash-hub.com/packages/htrnaseq/v0.3.0/components/workflows/runner)\
+    \ where a\nnumber of choices (input/output structure and location) have been made.\n\
+    \nInput for the workflow has to be `fastq` files (zipped or not). For bcl or other\
+    \ formats, please consider running\n[demultiplex](https://www.viash-hub.com/packages/demultiplex)\
+    \ first.\n"
+  info:
+    test_resources:
+    - path: "gs://viash-hub-resources/htrnaseq/v2"
+      dest: "resources_test"
+  viash_version: "0.9.4"
+  source: "src"
+  target: "target"
+  config_mods:
+  - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].config.script\
+    \ := 'includeConfig(\"nextflow_labels.config\")'\n.resources += {path: '/src/config/labels.config',\
+    \ dest: 'nextflow_labels.config'}\n.resources += {path: '/_viash.yaml', dest:\
+    \ '_viash.yaml'}\n"
+  - ".engines += { type: \"native\" }"
+  - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
+  - ".engines[.type == 'docker'].target_tag := 'update_craftbox'"
+  keywords:
+  - "bioinformatics"
+  - "sequencing"
+  - "high-throughput"
+  - "RNAseq"
+  - "mapping"
+  - "counting"
+  - "pipeline"
+  - "workflow"
+  license: "MIT"
+  organization: "vsh"
+  links:
+    repository: "https://github.com/viash-hub/htrnaseq"
+    issue_tracker: "https://github.com/viash-hub/htrnaseq/issues"

target/executable/parallel_map/_viash.yaml

@@ -0,0 +1,21 @@
+name: htrnaseq
+summary: |
+  A workflow for high-throughput RNA-seq data analyses.
+description: "This workflow is designed to process high-throughput RNA-seq data, where every\nwell of a microarray plate is a sample. A fasta file provided as input\ndefines the mapping between sample barcodes and wells.\n\nThe workflow is built in a modular fashion, where most of the base functionality\nis provided by components from [`biobox`](https://www.viash-hub.com/packages/biobox/latest)\nsupplemented by custom base components and workflow components in this package.\n\nThe full workflow is split in two major subworkflows that can be run independently:\n\n* **Well-demultiplexing:** Split the input (plate/pool level) fastq files per well.\n* **Mapping, counting and QC:** Run per-well mapping, counting and generate QC reports.\n\nEach of those can be started individually, or the full workflow can be run in two ways:\n\n1. Run the [main workflow](https://www.viash-hub.com/packages/htrnaseq/v0.3.0/components/workflows/htrnaseq) \ncontaining the main functionality.\n2. Run the [(opinionated) `runner`](https://www.viash-hub.com/packages/htrnaseq/v0.3.0/components/workflows/runner) where a\nnumber of choices (input/output structure and location) have been made.\n\nInput for the workflow has to be `fastq` files (zipped or not). For bcl or other formats, please consider running\n[demultiplex](https://www.viash-hub.com/packages/demultiplex) first.\n"
+license: MIT
+keywords: [bioinformatics, sequencing, high-throughput, RNAseq, mapping, counting, pipeline, workflow]
+links:
+  issue_tracker: https://github.com/viash-hub/htrnaseq/issues
+  repository: https://github.com/viash-hub/htrnaseq
+viash_version: 0.9.4
+info:
+  test_resources:
+    - path: gs://viash-hub-resources/htrnaseq/v2
+      dest: resources_test
+config_mods: |
+  .requirements.commands := ['ps']
+  .runners[.type == 'nextflow'].config.script := 'includeConfig("nextflow_labels.config")'
+  .resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}
+  .resources += {path: '/_viash.yaml', dest: '_viash.yaml'}
+version: update_craftbox
+organization: vsh

target/executable/parallel_map/nextflow_labels.config

@@ -0,0 +1,108 @@
+executor {
+  $k8s {
+    submitRateLimit = '10sec'
+    pollInterval = '1 sec'
+  }
+}
+
+process {
+  container = 'nextflow/bash:latest'
+  
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
+
+  // Retry for exit codes that have something to do with memory issues
+  errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
+  maxRetries = 3
+  maxMemory = 192.GB
+
+  // Resource labels
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
+  
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
+
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        withName: ".*parallel_map_process" {
+          maxForks = 1
+        }
+        maxMemory = 25.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 8 }
+  
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
+}
+
+def get_memory(to_compare) {
+    if (!process.containsKey("maxMemory") || !process.maxMemory) {
+      return to_compare
+    }
+
+    try {
+      if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
+        return process.maxMemory
+      }
+      else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
+        return max_memory as nextflow.util.MemoryUnit
+      }
+      else {
+        return to_compare
+      }  
+    } catch (all) {
+          println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
+          System.exit(1)
+    }
+  }