Build branch main with version main (d7d3b3e)

Build pipeline: viash-hub.demultiplex.main-6k4dk

Source commit: d7d3b3e1de

Source message: Automatically include resources labels during build (#32)

target/executable/io/interop_summary_to_csv/.config.vsh.yaml

@@ -38,6 +38,9 @@ resources:
 - type: "bash_script"
   path: "script.sh"
   is_executable: true
+- type: "file"
+  path: "nextflow_labels.config"
+  dest: "nextflow_labels.config"
 info: null
 status: "enabled"
 requirements:
@@ -109,6 +112,8 @@ runners:
       cpu200: "cpus = 200"
       cpu500: "cpus = 500"
       cpu1000: "cpus = 1000"
+    script:
+    - "includeConfig(\"nextflow_labels.config\")"
   debug: false
   container: "docker"
 engines:
@@ -141,9 +146,9 @@ build_info:
   output: "target/executable/io/interop_summary_to_csv"
   executable: "target/executable/io/interop_summary_to_csv/interop_summary_to_csv"
   viash_version: "0.9.0"
-  git_commit: "798e361afeea8dbb84dc23ef38e4fbc3883463e7"
-  git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex"
-  git_tag: "v0.1.1-13-g798e361"
+  git_commit: "d7d3b3e1de64f07a8b161b68a60098103ff691fb"
+  git_remote: "https://x-access-token:ghs_VevxXQvu4BBoh9Z4GlymTfE1DeHryU1sU0kQ@github.com/viash-hub/demultiplex"
+  git_tag: "v0.1.1-14-gd7d3b3e"
 package_config:
   name: "demultiplex"
   version: "main"
@@ -157,7 +162,9 @@ package_config:
   target: "target"
   config_mods:
   - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
-    \ := '$id'\n"
+    \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
+    .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
+    )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
   - ".engines[.type == 'docker'].target_tag := 'main'"

target/executable/io/interop_summary_to_csv/interop_summary_to_csv

@@ -470,9 +470,9 @@ tar -C /tmp/ --no-same-owner --no-same-permissions -xvf /tmp/interop.tar.gz && \
 mv /tmp/interop-1.3.1-Linux-GNU/bin/index-summary /tmp/interop-1.3.1-Linux-GNU/bin/summary /usr/local/bin/
 
 LABEL org.opencontainers.image.description="Companion container for running component io interop_summary_to_csv"
-LABEL org.opencontainers.image.created="2024-12-19T15:39:33Z"
+LABEL org.opencontainers.image.created="2024-12-20T11:16:00Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="798e361afeea8dbb84dc23ef38e4fbc3883463e7"
+LABEL org.opencontainers.image.revision="d7d3b3e1de64f07a8b161b68a60098103ff691fb"
 LABEL org.opencontainers.image.version="main"
 
 VIASHDOCKER

target/executable/io/interop_summary_to_csv/nextflow_labels.config

@@ -0,0 +1,98 @@
+process {
+  container = 'nextflow/bash:latest'
+  
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
+
+  // Retry for exit codes that have something to do with memory issues
+  errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
+  maxRetries = 3
+  maxMemory = 192.GB
+
+  // Resource labels
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
+  
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
+
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        maxMemory = 25.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 12 }
+  
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
+}
+
+def get_memory(to_compare) {
+    if (!process.containsKey("maxMemory") || !process.maxMemory) {
+      return to_compare
+    }
+
+    try {
+      if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
+        return process.maxMemory
+      }
+      else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
+        return max_memory as nextflow.util.MemoryUnit
+      }
+      else {
+        return to_compare
+      }  
+    } catch (all) {
+          println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
+          System.exit(1)
+    }
+  }

target/executable/io/publish/.config.vsh.yaml

@@ -94,6 +94,9 @@ resources:
 - type: "bash_script"
   path: "code.sh"
   is_executable: true
+- type: "file"
+  path: "nextflow_labels.config"
+  dest: "nextflow_labels.config"
 description: "Publish the processed results of the run"
 info: null
 status: "enabled"
@@ -166,6 +169,8 @@ runners:
       cpu200: "cpus = 200"
       cpu500: "cpus = 500"
       cpu1000: "cpus = 1000"
+    script:
+    - "includeConfig(\"nextflow_labels.config\")"
   debug: false
   container: "docker"
 engines:
@@ -191,9 +196,9 @@ build_info:
   output: "target/executable/io/publish"
   executable: "target/executable/io/publish/publish"
   viash_version: "0.9.0"
-  git_commit: "798e361afeea8dbb84dc23ef38e4fbc3883463e7"
-  git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex"
-  git_tag: "v0.1.1-13-g798e361"
+  git_commit: "d7d3b3e1de64f07a8b161b68a60098103ff691fb"
+  git_remote: "https://x-access-token:ghs_VevxXQvu4BBoh9Z4GlymTfE1DeHryU1sU0kQ@github.com/viash-hub/demultiplex"
+  git_tag: "v0.1.1-14-gd7d3b3e"
 package_config:
   name: "demultiplex"
   version: "main"
@@ -207,7 +212,9 @@ package_config:
   target: "target"
   config_mods:
   - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
-    \ := '$id'\n"
+    \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
+    .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
+    )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
   - ".engines[.type == 'docker'].target_tag := 'main'"

target/executable/io/publish/nextflow_labels.config

@@ -0,0 +1,98 @@
+process {
+  container = 'nextflow/bash:latest'
+  
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
+
+  // Retry for exit codes that have something to do with memory issues
+  errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
+  maxRetries = 3
+  maxMemory = 192.GB
+
+  // Resource labels
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
+  
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
+
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        maxMemory = 25.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 12 }
+  
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
+}
+
+def get_memory(to_compare) {
+    if (!process.containsKey("maxMemory") || !process.maxMemory) {
+      return to_compare
+    }
+
+    try {
+      if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
+        return process.maxMemory
+      }
+      else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
+        return max_memory as nextflow.util.MemoryUnit
+      }
+      else {
+        return to_compare
+      }  
+    } catch (all) {
+          println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
+          System.exit(1)
+    }
+  }

target/executable/io/publish/publish

@@ -490,9 +490,9 @@ RUN apt-get update && \
   rm -rf /var/lib/apt/lists/*
 
 LABEL org.opencontainers.image.description="Companion container for running component io publish"
-LABEL org.opencontainers.image.created="2024-12-19T15:39:33Z"
+LABEL org.opencontainers.image.created="2024-12-20T11:16:00Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="798e361afeea8dbb84dc23ef38e4fbc3883463e7"
+LABEL org.opencontainers.image.revision="d7d3b3e1de64f07a8b161b68a60098103ff691fb"
 LABEL org.opencontainers.image.version="main"
 
 VIASHDOCKER

target/executable/io/untar/.config.vsh.yaml

@@ -45,6 +45,9 @@ resources:
 - type: "bash_script"
   path: "script.sh"
   is_executable: true
+- type: "file"
+  path: "nextflow_labels.config"
+  dest: "nextflow_labels.config"
 description: "Unpack a .tar file. When the contents of the .tar file is just a single\
   \ directory,\nput the contents of the directory into the output folder instead of\
   \ that directory.\n"
@@ -123,6 +126,8 @@ runners:
       cpu200: "cpus = 200"
       cpu500: "cpus = 500"
       cpu1000: "cpus = 1000"
+    script:
+    - "includeConfig(\"nextflow_labels.config\")"
   debug: false
   container: "docker"
 engines:
@@ -148,9 +153,9 @@ build_info:
   output: "target/executable/io/untar"
   executable: "target/executable/io/untar/untar"
   viash_version: "0.9.0"
-  git_commit: "798e361afeea8dbb84dc23ef38e4fbc3883463e7"
-  git_remote: "https://x-access-token:ghs_44flLTQIYGBkc9xcwCZoLYe7mxXzgw1iszFO@github.com/viash-hub/demultiplex"
-  git_tag: "v0.1.1-13-g798e361"
+  git_commit: "d7d3b3e1de64f07a8b161b68a60098103ff691fb"
+  git_remote: "https://x-access-token:ghs_VevxXQvu4BBoh9Z4GlymTfE1DeHryU1sU0kQ@github.com/viash-hub/demultiplex"
+  git_tag: "v0.1.1-14-gd7d3b3e"
 package_config:
   name: "demultiplex"
   version: "main"
@@ -164,7 +169,9 @@ package_config:
   target: "target"
   config_mods:
   - ".requirements.commands := ['ps']\n.runners[.type == 'nextflow'].directives.tag\
-    \ := '$id'\n"
+    \ := '$id'\n.resources += {path: '/src/config/labels.config', dest: 'nextflow_labels.config'}\n\
+    .runners[.type == 'nextflow'].config.script := 'includeConfig(\"nextflow_labels.config\"\
+    )'\n"
   - ".engines += { type: \"native\" }"
   - ".engines[.type == 'docker'].target_registry := 'images.viash-hub.com'"
   - ".engines[.type == 'docker'].target_tag := 'main'"

target/executable/io/untar/nextflow_labels.config

@@ -0,0 +1,98 @@
+process {
+  container = 'nextflow/bash:latest'
+  
+  // default resources
+  memory = { 8.Gb * task.attempt }
+  cpus = 8
+  maxForks = 36
+
+  // Retry for exit codes that have something to do with memory issues
+  errorStrategy = { task.exitStatus in 137..140 ? 'retry' : 'terminate' }
+  maxRetries = 3
+  maxMemory = 192.GB
+
+  // Resource labels
+  withLabel: verylowcpu { cpus = 2 }
+  withLabel: lowcpu { cpus = 8 }
+  withLabel: midcpu { cpus = 16 }
+  withLabel: highcpu { cpus = 32 }
+  
+  withLabel: verylowmem { memory = { get_memory( 4.GB * task.attempt ) } }
+  withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+  withLabel: midmem { memory = { get_memory( 16.GB * task.attempt ) } }
+  withLabel: highmem { memory = { get_memory( 64.GB * task.attempt ) } }
+
+}
+
+profiles {
+  // detect tempdir
+  tempDir = java.nio.file.Paths.get(
+    System.getenv('NXF_TEMP') ?:
+      System.getenv('VIASH_TEMP') ?: 
+      System.getenv('TEMPDIR') ?: 
+      System.getenv('TMPDIR') ?: 
+      '/tmp'
+  ).toAbsolutePath()
+
+  mount_temp {
+    docker.temp            = tempDir
+    podman.temp            = tempDir
+    charliecloud.temp      = tempDir
+  }
+
+  no_publish {
+    process {
+      withName: '.*' {
+        publishDir = [
+          enabled: false
+        ]
+      }
+    }
+  }
+
+  docker {
+    docker.fixOwnership    = true
+    docker.enabled         = true
+    // docker.userEmulation   = true
+    singularity.enabled    = false
+    podman.enabled         = false
+    shifter.enabled        = false
+    charliecloud.enabled   = false
+  }
+
+  local {
+    // This config is for local processing.
+    process {
+        maxMemory = 25.GB
+        withLabel: verylowcpu { cpus = 2 }
+        withLabel: lowcpu { cpus = 4 }
+        withLabel: midcpu { cpus = 6 }
+        withLabel: highcpu { cpus = 12 }
+  
+        withLabel: lowmem { memory = { get_memory( 8.GB * task.attempt ) } }
+        withLabel: midmem { memory = { get_memory( 12.GB * task.attempt ) } }
+        withLabel: highmem { memory = { get_memory( 20.GB * task.attempt ) } }
+    }
+  }
+}
+
+def get_memory(to_compare) {
+    if (!process.containsKey("maxMemory") || !process.maxMemory) {
+      return to_compare
+    }
+
+    try {
+      if (process.containsKey("maxRetries") && process.maxRetries && task.attempt == (process.maxRetries as int)) {
+        return process.maxMemory
+      }
+      else if (to_compare.compareTo(process.maxMemory as nextflow.util.MemoryUnit) == 1) {
+        return max_memory as nextflow.util.MemoryUnit
+      }
+      else {
+        return to_compare
+      }  
+    } catch (all) {
+          println "Error processing memory resources. Please check that process.maxMemory '${process.maxMemory}' and process.maxRetries '${process.maxRetries}' are valid!"
+          System.exit(1)
+    }
+  }

target/executable/io/untar/untar

@@ -476,9 +476,9 @@ RUN apt-get update && \
   rm -rf /var/lib/apt/lists/*
 
 LABEL org.opencontainers.image.description="Companion container for running component io untar"
-LABEL org.opencontainers.image.created="2024-12-19T15:39:34Z"
+LABEL org.opencontainers.image.created="2024-12-20T11:16:01Z"
 LABEL org.opencontainers.image.source="https://github.com/viash-hub/demultiplex"
-LABEL org.opencontainers.image.revision="798e361afeea8dbb84dc23ef38e4fbc3883463e7"
+LABEL org.opencontainers.image.revision="d7d3b3e1de64f07a8b161b68a60098103ff691fb"
 LABEL org.opencontainers.image.version="main"
 
 VIASHDOCKER