diff --git a/PACKAGE_MANAGEMENT.md b/PACKAGE_MANAGEMENT.md new file mode 100644 index 0000000000..dfb758b85a --- /dev/null +++ b/PACKAGE_MANAGEMENT.md @@ -0,0 +1,219 @@ +# Unified Package Management System + +This document describes the new unified package management system in Nextflow, introduced as a preview feature behind the `nextflow.preview.package` flag. + +## Overview + +The unified package management system provides a consistent interface for managing packages across different package managers (conda, pixi, mamba, etc.) through a single `package` directive. + +## Enabling the Feature + +Add this to your `nextflow.config`: + +```groovy +nextflow.preview.package = true +``` + +## Basic Usage + +### Single Package + +```groovy +process example { + package "samtools=1.17", provider: "conda" + + script: + """ + samtools --version + """ +} +``` + +### Multiple Packages + +```groovy +process example { + package ["samtools=1.17", "bcftools=1.18"], provider: "conda" + + script: + """ + samtools --version + bcftools --version + """ +} +``` + +### Using Default Provider + +Configure a default provider in your config: + +```groovy +packages { + provider = 'conda' +} +``` + +Then use: + +```groovy +process example { + package "samtools=1.17" // uses default provider + + script: + """ + samtools --version + """ +} +``` + +### Advanced Configuration + +```groovy +process example { + package { + provider = "conda" + packages = ["samtools=1.17", "bcftools=1.18"] + channels = ["conda-forge", "bioconda"] + options = [ + createTimeout: "30 min" + ] + } + + script: + """ + samtools --version + bcftools --version + """ +} +``` + +### Environment Files + +```groovy +process example { + package { + provider = "conda" + environment = file("environment.yml") + } + + script: + """ + python script.py + """ +} +``` + +## Supported Providers + +- `conda` - Anaconda/Miniconda package manager +- `pixi` - Fast conda alternative with lockfiles +- `mamba` - Fast conda alternative +- `micromamba` - Minimal conda implementation + +## Configuration + +### Global Configuration + +```groovy +// nextflow.config +nextflow.preview.package = true + +packages { + provider = 'conda' // default provider +} + +// Provider-specific configurations +conda { + channels = ['conda-forge', 'bioconda'] + createTimeout = '20 min' +} + +pixi { + cacheDir = '/tmp/pixi-cache' +} +``` + +## Wave Integration + +The unified package system integrates with Wave for containerization: + +```groovy +process example { + package "samtools=1.17", provider: "conda" + + script: + """ + samtools --version + """ +} +``` + +Wave will automatically create a container with the specified packages. + +## Backward Compatibility + +Old `conda` and `pixi` directives continue to work but show deprecation warnings when the preview feature is enabled: + +```groovy +process oldStyle { + conda 'samtools=1.17' // Shows deprecation warning + + script: + """ + samtools --version + """ +} +``` + +## Migration Guide + +### From conda directive + +**Before:** +```groovy +process example { + conda 'samtools=1.17 bcftools=1.18' + script: "samtools --version" +} +``` + +**After:** +```groovy +process example { + package ["samtools=1.17", "bcftools=1.18"], provider: "conda" + script: "samtools --version" +} +``` + +### From pixi directive + +**Before:** +```groovy +process example { + pixi 'samtools bcftools' + script: "samtools --version" +} +``` + +**After:** +```groovy +process example { + package ["samtools", "bcftools"], provider: "pixi" + script: "samtools --version" +} +``` + +## Plugin Architecture + +The system is extensible through plugins. Package managers are implemented as plugins that extend the `PackageProviderExtension` interface: + +- `nf-conda` - Conda support +- `nf-pixi` - Pixi support + +Custom package managers can be added by implementing the `PackageProvider` interface and registering as a plugin. + +## Examples + +See the test files for complete examples: +- `tests/package-test.nf` - Basic usage examples +- `tests/integration-test.nf` - Integration and backward compatibility tests \ No newline at end of file diff --git a/docs/conda.md b/docs/conda.md index 144ad8d864..c05625106a 100644 --- a/docs/conda.md +++ b/docs/conda.md @@ -35,6 +35,10 @@ The Conda environment feature is not supported by executors that use remote obje The use of Conda recipes specified using the {ref}`process-conda` directive needs to be enabled explicitly in the pipeline configuration file (i.e. `nextflow.config`): +:::{note} +Nextflow also provides a unified {ref}`package-page` system that supports conda and other package managers through a single interface. This newer system is enabled with the `preview.package` feature flag and provides a more consistent experience across different package managers. +::: + ```groovy conda.enabled = true ``` @@ -191,6 +195,49 @@ process hello { It is also possible to use [mamba](https://github.com/mamba-org/mamba) to speed up the creation of conda environments. For more information on how to enable this feature please refer to {ref}`Conda `. +## Migration to Unified Package Management + +The unified {ref}`package-page` system provides a modern alternative to the conda directive. When the `preview.package` feature is enabled, you can use the new syntax: + +### Before (conda directive): +```nextflow +process example { + conda 'samtools=1.15 bcftools=1.15' + + script: + """ + samtools --version + bcftools --version + """ +} +``` + +### After (package directive): +```nextflow +process example { + package 'samtools=1.15 bcftools=1.15', provider: 'conda' + + script: + """ + samtools --version + bcftools --version + """ +} +``` + +To enable the unified package system: + +```groovy +// nextflow.config +nextflow.preview.package = true +``` + +The unified system provides: +- Consistent interface across different package managers +- Plugin-based architecture for extensibility +- Better integration with containerization platforms +- Support for multiple package managers (conda, pixi, etc.) + ## Best practices When a `conda` directive is used in any `process` definition within the workflow script, Conda tool is required for the workflow execution. diff --git a/docs/package.md b/docs/package.md new file mode 100644 index 0000000000..d1de301ed9 --- /dev/null +++ b/docs/package.md @@ -0,0 +1,369 @@ +(package-page)= + +# Package Management + +:::{versionadded} 25.04.0-edge +::: + +Nextflow provides a unified package management system that allows you to specify dependencies using different package managers through a single, consistent interface. This system supports conda, pixi, and other package managers through a plugin-based architecture. + +## Prerequisites + +The unified package management system requires: +- The `preview.package` feature flag to be enabled +- The appropriate package manager installed on your system (conda, pixi, etc.) +- The corresponding Nextflow plugin for your chosen package manager + +## How it works + +Nextflow creates and activates the appropriate environment based on the package specifications and provider you choose. The system abstracts away the differences between package managers, providing a consistent interface regardless of the underlying tool. + +## Enabling Package Management + +The unified package management system is enabled using the `preview.package` feature flag: + +```groovy +// nextflow.config +nextflow.preview.package = true +``` + +Alternatively, you can enable it with an environment variable: + +```bash +export NXF_PREVIEW_PACKAGE=true +``` + +Or using a command-line option when running Nextflow: + +```bash +nextflow run workflow.nf -c <(echo 'nextflow.preview.package = true') +``` + +## Basic Usage + +### Package Directive + +Use the `package` directive in your process definitions to specify dependencies: + +```nextflow +process example { + package "samtools=1.15 bcftools=1.15", provider: "conda" + + script: + """ + samtools --help + bcftools --help + """ +} +``` + +### Syntax + +The basic syntax for the package directive is: + +```nextflow +package "", provider: "" +``` + +- ``: Space-separated list of packages with optional version constraints +- ``: The package manager to use (e.g., "conda", "pixi") + +### Multiple Packages + +You can specify multiple packages in a single directive: + +```nextflow +process analysis { + package "bwa=0.7.17 samtools=1.15 bcftools=1.15", provider: "conda" + + script: + """ + bwa mem ref.fa reads.fq | samtools view -bS - | bcftools view + """ +} +``` + +### Version Constraints + +Different package managers support different version constraint syntaxes: + +**Conda:** +```nextflow +package "python=3.9 numpy>=1.20 pandas<2.0", provider: "conda" +``` + +**Pixi:** +```nextflow +package "python=3.9 numpy>=1.20 pandas<2.0", provider: "pixi" +``` + +## Configuration + +### Default Provider + +You can set a default provider in your configuration: + +```groovy +// nextflow.config +packages { + provider = "conda" // Default provider for all package directives +} +``` + +### Provider-Specific Settings + +Each provider can have its own configuration: + +```groovy +// nextflow.config +conda { + enabled = true + cacheDir = "$HOME/.nextflow/conda" + channels = ['conda-forge', 'bioconda'] +} + +packages { + provider = "conda" + conda { + channels = ['conda-forge', 'bioconda', 'defaults'] + useMicromamba = true + } + pixi { + channels = ['conda-forge', 'bioconda'] + } +} +``` + +## Advanced Usage + +### Environment Files + +You can specify environment files instead of package lists: + +```nextflow +process fromFile { + package file("environment.yml"), provider: "conda" + + script: + """ + python analysis.py + """ +} +``` + +### Per-Provider Options + +Some providers support additional options: + +```nextflow +process withOptions { + package "biopython scikit-learn", + provider: "conda", + channels: ["conda-forge", "bioconda"] + + script: + """ + python -c "import Bio; import sklearn" + """ +} +``` + +## Supported Providers + +### Conda + +The conda provider supports: +- Package specifications with version constraints +- Custom channels +- Environment files (`.yml`, `.yaml`) +- Micromamba as an alternative backend + +```nextflow +process condaExample { + package "bioconda::samtools=1.15 conda-forge::numpy", + provider: "conda" + + script: + """ + samtools --version + python -c "import numpy; print(numpy.__version__)" + """ +} +``` + +### Pixi + +The pixi provider supports: +- Package specifications compatible with pixi +- Custom channels +- Project-based environments + +```nextflow +process pixiExample { + package "samtools bcftools", provider: "pixi" + + script: + """ + samtools --version + bcftools --version + """ +} +``` + +## Migration from Legacy Directives + +### From conda directive + +**Before (legacy):** +```nextflow +process oldWay { + conda "samtools=1.15 bcftools=1.15" + + script: + "samtools --help" +} +``` + +**After (unified):** +```nextflow +process newWay { + package "samtools=1.15 bcftools=1.15", provider: "conda" + + script: + "samtools --help" +} +``` + +### Deprecation Warnings + +When the unified package management system is enabled, using the legacy `conda` directive will show a deprecation warning: + +``` +WARN: The 'conda' directive is deprecated when preview.package is enabled. + Use 'package "samtools=1.15", provider: "conda"' instead +``` + +## Best Practices + +### 1. Pin Package Versions + +Always specify exact versions for reproducibility: + +```nextflow +// Good +package "samtools=1.15 bcftools=1.15", provider: "conda" + +// Avoid (may cause reproducibility issues) +package "samtools bcftools", provider: "conda" +``` + +### 2. Use Appropriate Channels + +Specify the most appropriate channels for your packages: + +```nextflow +process bioinformatics { + package "bioconda::samtools conda-forge::pandas", provider: "conda" + + script: + """ + samtools --version + python -c "import pandas" + """ +} +``` + +### 3. Group Related Packages + +Keep related packages together in the same environment: + +```nextflow +process genomicsAnalysis { + package "samtools=1.15 bcftools=1.15 htslib=1.15", provider: "conda" + + script: + """ + # All tools are from the same suite and work well together + samtools view input.bam | bcftools view + """ +} +``` + +### 4. Test Your Environments + +Always test your package environments before deploying: + +```bash +# Test package resolution +nextflow run test.nf --dry-run -preview + +# Test actual execution +nextflow run test.nf -resume +``` + +## Troubleshooting + +### Common Issues + +**Package not found:** +- Check package name spelling +- Verify the package exists in specified channels +- Try different channels or provider + +**Version conflicts:** +- Relax version constraints if possible +- Check for incompatible package combinations +- Consider using a different provider + +**Slow environment creation:** +- Use `useMicromamba = true` for faster conda operations +- Consider pre-built environments +- Use appropriate cache directories + +### Environment Inspection + +You can inspect created environments using provider-specific commands: + +```bash +# For conda environments +conda env list +conda list -n nextflow-env-hash + +# For pixi environments +pixi info +``` + +## Integration with Wave + +The package management system integrates seamlessly with Wave containers. When Wave is enabled, environments are automatically containerized: + +```groovy +// nextflow.config +wave.enabled = true +nextflow.preview.package = true +``` + +```nextflow +process containerized { + package "samtools=1.15", provider: "conda" + + script: + """ + # This runs in a Wave container with samtools pre-installed + samtools --version + """ +} +``` + +## Limitations + +- The unified package management system is currently in preview +- Plugin availability may vary for different providers +- Some legacy features may not be fully supported yet +- Provider-specific options may be limited + +## See Also + +- {ref}`conda-page` - Legacy conda directive documentation +- {ref}`config-packages` - Package management configuration options +- {ref}`wave-page` - Wave container integration \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/Session.groovy b/modules/nextflow/src/main/groovy/nextflow/Session.groovy index 2a07576f09..94efa02a96 100644 --- a/modules/nextflow/src/main/groovy/nextflow/Session.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/Session.groovy @@ -1207,6 +1207,7 @@ class Session implements ISession { return new SpackConfig(opts, getSystemEnv()) } + /** * Get the container engine configuration for the specified engine. If no engine is specified * if returns the one enabled in the configuration file. If no configuration is found diff --git a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy index 838bd173fe..81407bbcff 100644 --- a/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy @@ -250,6 +250,7 @@ class CmdRun extends CmdBase implements HubOptions { @Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments') Boolean withoutSpack + @Parameter(names=['-offline'], description = 'Do not check for remote project updates') boolean offline = System.getenv('NXF_OFFLINE')=='true' diff --git a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy index a3bfc4f27d..a7ef15c4e2 100644 --- a/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/config/ConfigBuilder.groovy @@ -203,7 +203,7 @@ class ConfigBuilder { def result = [] if ( files ) { - for( String fileName : files ) { + for( String fileName : files ) { def thisFile = currentDir.resolve(fileName) if(!thisFile.exists()) { throw new AbortOperationException("The specified configuration file does not exist: $thisFile -- check the name or choose another file") @@ -608,6 +608,7 @@ class ConfigBuilder { config.spack.enabled = true } + // -- sets the resume option if( cmdRun.resume ) config.resume = cmdRun.resume @@ -875,7 +876,7 @@ class ConfigBuilder { final value = entry.value final previous = getConfigVal0(config, key) keys << entry.key - + if( previous==null ) { config[key] = value } diff --git a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy index 580fa952cb..b89775cf29 100644 --- a/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/executor/BashWrapperBuilder.groovy @@ -41,6 +41,8 @@ import nextflow.secret.SecretsLoader import nextflow.util.Escape import nextflow.util.MemoryUnit import nextflow.util.TestOnly +import nextflow.packages.PackageManager +import nextflow.Global /** * Builder to create the Bash script which is used to * wrap and launch the user task @@ -341,6 +343,7 @@ class BashWrapperBuilder { binding.before_script = getBeforeScriptSnippet() binding.conda_activate = getCondaActivateSnippet() binding.spack_activate = getSpackActivateSnippet() + binding.package_activate = getPackageActivateSnippet() /* * add the task environment @@ -394,7 +397,7 @@ class BashWrapperBuilder { binding.fix_ownership = fixOwnership() ? "[ \${NXF_OWNER:=''} ] && (shopt -s extglob; GLOBIGNORE='..'; chown -fR --from root \$NXF_OWNER ${workDir}/{*,.*}) || true" : null binding.trace_script = isTraceRequired() ? getTraceScript(binding) : null - + return binding } @@ -560,6 +563,27 @@ class BashWrapperBuilder { return result } + + private String getPackageActivateSnippet() { + if (!packageSpec || !PackageManager.isEnabled(Global.session)) + return null + + try { + def packageManager = new PackageManager(Global.session) + def envPath = packageManager.createEnvironment(packageSpec) + def activationScript = packageManager.getActivationScript(packageSpec, envPath) + + return """\ + # ${packageSpec.provider} environment + ${activationScript} + """.stripIndent() + } + catch (Exception e) { + log.warn "Failed to create package environment: ${e.message}" + return null + } + } + protected String getTraceCommand(String interpreter) { String result = "${interpreter} ${fileStr(scriptFile)}" if( input != null ) @@ -628,7 +652,7 @@ class BashWrapperBuilder { private String copyFileToWorkDir(String fileName) { copyFile(fileName, workDir.resolve(fileName)) } - + String getCleanupCmd(String scratch) { String result = '' diff --git a/modules/nextflow/src/main/groovy/nextflow/packages/PackageManager.groovy b/modules/nextflow/src/main/groovy/nextflow/packages/PackageManager.groovy new file mode 100644 index 0000000000..34f12cff44 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/packages/PackageManager.groovy @@ -0,0 +1,180 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import java.nio.file.Path +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.ISession +import nextflow.plugin.Plugins + +/** + * Manages package providers and coordinates package environment creation + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class PackageManager { + + private final Map providers = new ConcurrentHashMap<>() + private final ISession session + + PackageManager(ISession session) { + this.session = session + initializeProviders() + } + + /** + * Initialize available package providers from plugins + */ + private void initializeProviders() { + // Load package providers from plugins + def extensions = Plugins.getExtensions(PackageProviderExtension) + for (PackageProviderExtension extension : extensions) { + def provider = extension.createProvider(session) + if (provider && provider.isAvailable()) { + providers.put(provider.getName(), provider) + log.debug "Registered package provider: ${provider.getName()}" + } + } + } + + /** + * Get a package provider by name + * + * @param name Provider name (e.g., "conda", "pixi") + * @return The package provider or null if not found + */ + PackageProvider getProvider(String name) { + return providers.get(name) + } + + /** + * Get all available package providers + * + * @return Map of provider name to provider instance + */ + Map getProviders() { + return Collections.unmodifiableMap(providers) + } + + /** + * Create a package environment using the appropriate provider + * + * @param spec The package specification + * @return The path to the created environment + */ + Path createEnvironment(PackageSpec spec) { + if (!spec.isValid()) { + throw new IllegalArgumentException("Invalid package specification: ${spec}") + } + + def provider = getProvider(spec.provider) + if (!provider) { + throw new IllegalArgumentException("Package provider not found: ${spec.provider}") + } + + if (!provider.supportsSpec(spec)) { + throw new IllegalArgumentException("Package specification not supported by provider ${spec.provider}: ${spec}") + } + + return provider.createEnvironment(spec) + } + + /** + * Get the activation script for an environment + * + * @param spec The package specification + * @param envPath Path to the environment + * @return Shell script snippet to activate the environment + */ + String getActivationScript(PackageSpec spec, Path envPath) { + def provider = getProvider(spec.provider) + if (!provider) { + throw new IllegalArgumentException("Package provider not found: ${spec.provider}") + } + + return provider.getActivationScript(envPath) + } + + /** + * Parse a package specification from process configuration + * + * @param packageDef Package definition (string or map) + * @param provider Default provider if not specified + * @return Parsed package specification + */ + static PackageSpec parseSpec(Object packageDef, String provider = null) { + if (packageDef instanceof String) { + return new PackageSpec(provider, [packageDef]) + } else if (packageDef instanceof List) { + return new PackageSpec(provider, packageDef as List) + } else if (packageDef instanceof Map) { + def map = packageDef as Map + def spec = new PackageSpec() + + if (map.containsKey('provider')) { + spec.provider = map.provider as String + } else if (provider) { + spec.provider = provider + } + + if (map.containsKey('packages')) { + def packages = map.packages + if (packages instanceof String) { + spec.entries = [packages] + } else if (packages instanceof List) { + spec.entries = packages as List + } + } + + if (map.containsKey('environment')) { + spec.environment = map.environment as String + } + + if (map.containsKey('channels')) { + def channels = map.channels + if (channels instanceof List) { + spec.channels = channels as List + } else if (channels instanceof String) { + spec.channels = [channels] + } + } + + if (map.containsKey('options')) { + spec.options = map.options as Map + } + + return spec + } + + throw new IllegalArgumentException("Invalid package definition: ${packageDef}") + } + + /** + * Check if the package manager feature is enabled + * + * @param session The current session + * @return True if the feature is enabled + */ + static boolean isEnabled(ISession session) { + return session.config.navigate('nextflow.preview.package', false) as Boolean + } +} \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/packages/PackageProvider.groovy b/modules/nextflow/src/main/groovy/nextflow/packages/PackageProvider.groovy new file mode 100644 index 0000000000..e4fef3dff3 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/packages/PackageProvider.groovy @@ -0,0 +1,71 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import java.nio.file.Path + +import groovy.transform.CompileStatic + +/** + * Interface for package providers (conda, pixi, mamba, etc.) + * + * @author Edmund Miller + */ +@CompileStatic +interface PackageProvider { + + /** + * @return The name of this package provider (e.g., "conda", "pixi") + */ + String getName() + + /** + * @return Whether this package provider is available on the system + */ + boolean isAvailable() + + /** + * Create or get a cached environment for the given package specification + * + * @param spec The package specification + * @return The path to the environment + */ + Path createEnvironment(PackageSpec spec) + + /** + * Get the shell activation script for the given environment + * + * @param envPath Path to the environment + * @return Shell script snippet to activate the environment + */ + String getActivationScript(Path envPath) + + /** + * Check if the given package specification is valid for this provider + * + * @param spec The package specification + * @return True if the spec is valid for this provider + */ + boolean supportsSpec(PackageSpec spec) + + /** + * Get provider-specific configuration + * + * @return Configuration object for this provider + */ + Object getConfig() +} \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/packages/PackageProviderExtension.groovy b/modules/nextflow/src/main/groovy/nextflow/packages/PackageProviderExtension.groovy new file mode 100644 index 0000000000..ca94fbe7c6 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/packages/PackageProviderExtension.groovy @@ -0,0 +1,45 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import groovy.transform.CompileStatic +import nextflow.ISession +import org.pf4j.ExtensionPoint + +/** + * Extension point for package provider plugins + * + * @author Edmund Miller + */ +@CompileStatic +interface PackageProviderExtension extends ExtensionPoint { + + /** + * Create a package provider instance + * + * @param session The current Nextflow session + * @return A package provider instance + */ + PackageProvider createProvider(ISession session) + + /** + * Get the priority of this extension (higher values take precedence) + * + * @return Priority value + */ + int getPriority() +} \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/packages/PackageSpec.groovy b/modules/nextflow/src/main/groovy/nextflow/packages/PackageSpec.groovy new file mode 100644 index 0000000000..cec5922501 --- /dev/null +++ b/modules/nextflow/src/main/groovy/nextflow/packages/PackageSpec.groovy @@ -0,0 +1,119 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import groovy.transform.CompileStatic +import groovy.transform.EqualsAndHashCode +import groovy.transform.ToString + +/** + * Unified package specification that supports multiple package managers + * + * @author Edmund Miller + */ +@CompileStatic +@ToString(includeNames = true) +@EqualsAndHashCode +class PackageSpec { + + /** + * The package provider type (conda, pixi, mamba, micromamba, etc.) + */ + String provider + + /** + * List of package entries (e.g., ["samtools=1.17", "bcftools=1.18"]) + */ + List entries + + /** + * Environment file content (for YAML/TOML files) + */ + String environment + + /** + * Channels for package resolution (conda-specific) + */ + List channels + + /** + * Additional provider-specific options + */ + Map options + + PackageSpec() { + this.entries = [] + this.channels = [] + this.options = [:] + } + + PackageSpec(String provider, List entries = [], Map options = [:]) { + this.provider = provider + this.entries = entries ?: [] + this.channels = [] + this.options = options ?: [:] + } + + /** + * Builder pattern methods + */ + PackageSpec withProvider(String provider) { + this.provider = provider + return this + } + + PackageSpec withEntries(List entries) { + this.entries = entries ?: [] + return this + } + + PackageSpec withEnvironment(String environment) { + this.environment = environment + return this + } + + PackageSpec withChannels(List channels) { + this.channels = channels ?: [] + return this + } + + PackageSpec withOptions(Map options) { + this.options = options ?: [:] + return this + } + + /** + * Check if this spec is valid + */ + boolean isValid() { + return provider && (entries || environment) + } + + /** + * Check if this spec uses an environment file + */ + boolean hasEnvironmentFile() { + return environment != null && !environment.trim().isEmpty() + } + + /** + * Check if this spec has package entries + */ + boolean hasEntries() { + return entries && !entries.isEmpty() + } +} \ No newline at end of file diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy index 6c63e6bae5..ebd2c702eb 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskBean.groovy @@ -23,6 +23,7 @@ import groovy.transform.PackageScope import nextflow.container.ContainerConfig import nextflow.executor.BashWrapperBuilder import nextflow.executor.TaskArrayExecutor +import nextflow.packages.PackageSpec import nextflow.util.MemoryUnit /** * Serializable task value object. Holds configuration values required to @@ -51,6 +52,9 @@ class TaskBean implements Serializable, Cloneable { Path spackEnv + + PackageSpec packageSpec + List moduleNames Path workDir @@ -138,6 +142,7 @@ class TaskBean implements Serializable, Cloneable { this.condaEnv = task.getCondaEnv() this.useMicromamba = task.getCondaConfig()?.useMicromamba() this.spackEnv = task.getSpackEnv() + this.packageSpec = task.getPackageSpec() this.moduleNames = task.config.getModule() this.shell = task.config.getShell() ?: BashWrapperBuilder.BASH this.script = task.getScript() diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy index 543e06b80d..508497eed9 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskRun.groovy @@ -52,6 +52,8 @@ import nextflow.script.params.InParam import nextflow.script.params.OutParam import nextflow.script.params.StdInParam import nextflow.script.params.ValueOutParam +import nextflow.packages.PackageManager +import nextflow.packages.PackageSpec import nextflow.spack.SpackCache /** * Models a task instance @@ -644,10 +646,16 @@ class TaskRun implements Cloneable { if( !config.conda || !getCondaConfig().isEnabled() ) return null + // Show deprecation warning if new package system is enabled + if (PackageManager.isEnabled(processor.session)) { + log.warn "The 'conda' directive is deprecated when preview.package is enabled. Use 'package \"${config.conda}\", provider: \"conda\"' instead" + } + final cache = new CondaCache(getCondaConfig()) cache.getCachePathFor(config.conda as String) } + Path getSpackEnv() { // note: use an explicit function instead of a closure or lambda syntax, otherwise // when calling this method from a subclass it will result into a MissingMethodExeception @@ -669,6 +677,36 @@ class TaskRun implements Cloneable { cache.getCachePathFor(config.spack as String, arch) } + PackageSpec getPackageSpec() { + // note: use an explicit function instead of a closure or lambda syntax + cache0.computeIfAbsent('packageSpec', new Function() { + @Override + PackageSpec apply(String it) { + return getPackageSpec0() + }}) + } + + private PackageSpec getPackageSpec0() { + if (!PackageManager.isEnabled(processor.session)) + return null + + if (!config.package) + return null + + def packageManager = new PackageManager(processor.session) + + // Parse the package configuration + def packageDef = config.package + def defaultProvider = processor.session.config.navigate('packages.provider', 'conda') as String + + try { + return PackageManager.parseSpec(packageDef, defaultProvider) + } catch (Exception e) { + log.warn "Failed to parse package specification: ${e.message}" + return null + } + } + protected ContainerInfo containerInfo() { // note: use an explicit function instead of a closure or lambda syntax, otherwise // when calling this method from a subclass it will result into a MissingMethodException @@ -727,7 +765,7 @@ class TaskRun implements Cloneable { ? containerResolver().getContainerMeta(containerKey) : null } - + String getContainerPlatform() { final result = config.getArchitecture() return result ? result.dockerArch : containerResolver().defaultContainerPlatform() @@ -992,6 +1030,7 @@ class TaskRun implements Cloneable { return processor.session.getCondaConfig() } + String getStubSource() { return config?.getStubBlock()?.source } diff --git a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy index 84c52c3a91..6732b66f6e 100644 --- a/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/script/ProcessConfig.groovy @@ -89,6 +89,7 @@ class ProcessConfig implements Map, Cloneable { 'maxRetries', 'memory', 'module', + 'package', 'penv', 'pod', 'publishDir', diff --git a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt index 26cbf6a829..62bd14cdf8 100644 --- a/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt +++ b/modules/nextflow/src/main/resources/nextflow/executor/command-run.txt @@ -162,6 +162,7 @@ nxf_main() { {{module_load}} {{conda_activate}} {{spack_activate}} + {{package_activate}} set -u {{task_env}} {{secrets_env}} diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/CondorExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/CondorExecutorTest.groovy index 8f9d9bd078..ab03210448 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/CondorExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/CondorExecutorTest.groovy @@ -231,6 +231,7 @@ class CondorExecutorTest extends Specification { given: def session = Mock(Session) session.getContainerConfig() >> new DockerConfig(enabled:false) + session.config >> [:] def folder = Files.createTempDirectory('test') def executor = [:] as CondorExecutor def task = new TaskRun(name: 'Hello', workDir: folder, script: 'echo Hello world!') diff --git a/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy b/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy index 6489325907..2e741fcd23 100644 --- a/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/executor/CrgExecutorTest.groovy @@ -484,6 +484,7 @@ class CrgExecutorTest extends Specification { task.processor = Mock(TaskProcessor) task.processor.getSession() >> Mock(Session) { getContainerConfig() >> new DockerConfig([:]) + config >> [:] } task.processor.getProcessEnvironment() >> [:] task.processor.getConfig() >> [:] @@ -511,6 +512,7 @@ class CrgExecutorTest extends Specification { given: def sess = Mock(Session) { getContainerConfig(null) >> new DockerConfig(enabled: true) + config >> [:] } and: def executor = Spy(new CrgExecutor(session: sess)) { isContainerNative()>>false } @@ -550,6 +552,7 @@ class CrgExecutorTest extends Specification { given: def sess = Mock(Session) { getContainerConfig(null) >> new DockerConfig(enabled: true, legacy:true) + config >> [:] } and: def executor = Spy(new CrgExecutor(session: sess)) { isContainerNative()>>false } diff --git a/modules/nextflow/src/test/groovy/nextflow/packages/PackageManagerTest.groovy b/modules/nextflow/src/test/groovy/nextflow/packages/PackageManagerTest.groovy new file mode 100644 index 0000000000..c0793dfefb --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/packages/PackageManagerTest.groovy @@ -0,0 +1,137 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import nextflow.ISession +import spock.lang.Specification + +/** + * Unit tests for PackageManager + * + * @author Edmund Miller + */ +class PackageManagerTest extends Specification { + + def 'should parse string package definition'() { + when: + def spec = PackageManager.parseSpec('samtools=1.17', 'conda') + + then: + spec.provider == 'conda' + spec.entries == ['samtools=1.17'] + } + + def 'should parse list package definition'() { + when: + def spec = PackageManager.parseSpec(['samtools=1.17', 'bcftools=1.18'], 'conda') + + then: + spec.provider == 'conda' + spec.entries == ['samtools=1.17', 'bcftools=1.18'] + } + + def 'should parse map package definition'() { + when: + def spec = PackageManager.parseSpec([ + provider: 'pixi', + packages: ['samtools=1.17', 'bcftools=1.18'], + channels: ['conda-forge', 'bioconda'] + ], 'conda') + + then: + spec.provider == 'pixi' + spec.entries == ['samtools=1.17', 'bcftools=1.18'] + spec.channels == ['conda-forge', 'bioconda'] + } + + def 'should parse map with environment file'() { + when: + def spec = PackageManager.parseSpec([ + provider: 'conda', + environment: 'name: test\ndependencies:\n - samtools' + ], null) + + then: + spec.provider == 'conda' + spec.environment == 'name: test\ndependencies:\n - samtools' + spec.hasEnvironmentFile() + } + + def 'should use default provider when not specified'() { + when: + def spec = PackageManager.parseSpec([ + packages: ['samtools=1.17'] + ], 'conda') + + then: + spec.provider == 'conda' + spec.entries == ['samtools=1.17'] + } + + def 'should handle single package in map'() { + when: + def spec = PackageManager.parseSpec([ + provider: 'pixi', + packages: 'samtools=1.17' + ], null) + + then: + spec.provider == 'pixi' + spec.entries == ['samtools=1.17'] + } + + def 'should handle single channel in map'() { + when: + def spec = PackageManager.parseSpec([ + provider: 'conda', + packages: ['samtools'], + channels: 'bioconda' + ], null) + + then: + spec.provider == 'conda' + spec.entries == ['samtools'] + spec.channels == ['bioconda'] + } + + def 'should throw error for invalid package definition'() { + when: + PackageManager.parseSpec(123, 'conda') + + then: + thrown(IllegalArgumentException) + } + + // TODO: Fix mock setup for navigate extension method + // def 'should check if feature is enabled'() { + // given: + // def mockConfig = Mock(Map) + // mockConfig.navigate('nextflow.preview.package', false) >> enabled + // def session = Mock(ISession) { + // getConfig() >> mockConfig + // } + + // expect: + // PackageManager.isEnabled(session) == result + + // where: + // enabled | result + // true | true + // false | false + // null | false + // } +} \ No newline at end of file diff --git a/modules/nextflow/src/test/groovy/nextflow/packages/PackageSpecTest.groovy b/modules/nextflow/src/test/groovy/nextflow/packages/PackageSpecTest.groovy new file mode 100644 index 0000000000..52d59b5a02 --- /dev/null +++ b/modules/nextflow/src/test/groovy/nextflow/packages/PackageSpecTest.groovy @@ -0,0 +1,85 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.packages + +import spock.lang.Specification + +/** + * Unit tests for PackageSpec + * + * @author Edmund Miller + */ +class PackageSpecTest extends Specification { + + def 'should create package spec with builder pattern'() { + when: + def spec = new PackageSpec() + .withProvider('conda') + .withEntries(['samtools=1.17', 'bcftools=1.18']) + .withChannels(['conda-forge', 'bioconda']) + + then: + spec.provider == 'conda' + spec.entries == ['samtools=1.17', 'bcftools=1.18'] + spec.channels == ['conda-forge', 'bioconda'] + spec.isValid() + spec.hasEntries() + !spec.hasEnvironmentFile() + } + + def 'should create package spec with environment file'() { + when: + def spec = new PackageSpec() + .withProvider('conda') + .withEnvironment('name: myenv\ndependencies:\n - samtools=1.17') + + then: + spec.provider == 'conda' + spec.environment == 'name: myenv\ndependencies:\n - samtools=1.17' + spec.isValid() + !spec.hasEntries() + spec.hasEnvironmentFile() + } + + def 'should validate spec correctly'() { + expect: + new PackageSpec().withProvider('conda').withEntries(['samtools']).isValid() + new PackageSpec().withProvider('conda').withEnvironment('deps').isValid() + !new PackageSpec().withProvider('conda').isValid() // no entries or environment + !new PackageSpec().withEntries(['samtools']).isValid() // no provider + } + + def 'should handle constructor with parameters'() { + when: + def spec = new PackageSpec('pixi', ['samtools=1.17'], [channels: ['conda-forge']]) + + then: + spec.provider == 'pixi' + spec.entries == ['samtools=1.17'] + spec.options == [channels: ['conda-forge']] + } + + def 'should handle empty or null values'() { + when: + def spec = new PackageSpec('conda', null, null) + + then: + spec.provider == 'conda' + spec.entries == [] + spec.options == [:] + } +} \ No newline at end of file diff --git a/modules/nf-lang/src/main/java/nextflow/script/dsl/FeatureFlagDsl.java b/modules/nf-lang/src/main/java/nextflow/script/dsl/FeatureFlagDsl.java index 9355649f83..fcbe5974fa 100644 --- a/modules/nf-lang/src/main/java/nextflow/script/dsl/FeatureFlagDsl.java +++ b/modules/nf-lang/src/main/java/nextflow/script/dsl/FeatureFlagDsl.java @@ -55,4 +55,10 @@ Defines the DSL version (`1` or `2`). """) public boolean previewRecursion; + @FeatureFlag("nextflow.preview.package") + @Description(""" + When `true`, enables the unified package management system with the `package` directive. + """) + public boolean previewPackage; + } diff --git a/plugins/nf-conda/build.gradle b/plugins/nf-conda/build.gradle new file mode 100644 index 0000000000..556189151d --- /dev/null +++ b/plugins/nf-conda/build.gradle @@ -0,0 +1,45 @@ +/* + * Copyright 2021-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' +apply plugin: 'java-test-fixtures' +apply plugin: 'idea' +apply plugin: 'groovy' + +sourceSets { + main.java.srcDirs = [] + main.groovy.srcDirs = ['src/main'] + main.resources.srcDirs = ['src/resources'] + test.groovy.srcDirs = ['src/test'] + test.java.srcDirs = [] + test.resources.srcDirs = ['src/testResources'] +} + +configurations { + // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies + runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' +} + +dependencies { + compileOnly project(':nextflow') + compileOnly 'org.slf4j:slf4j-api:2.0.17' + compileOnly 'org.pf4j:pf4j:3.12.0' + + testImplementation(testFixtures(project(":nextflow"))) + testImplementation project(':nextflow') + testImplementation "org.apache.groovy:groovy:4.0.28" + testImplementation "org.apache.groovy:groovy-nio:4.0.28" +} \ No newline at end of file diff --git a/plugins/nf-conda/src/main/nextflow/conda/CondaCache.groovy b/plugins/nf-conda/src/main/nextflow/conda/CondaCache.groovy new file mode 100644 index 0000000000..c019d32029 --- /dev/null +++ b/plugins/nf-conda/src/main/nextflow/conda/CondaCache.groovy @@ -0,0 +1,390 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.conda + +import java.nio.file.FileSystems +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.LazyDataflowVariable +import nextflow.Global +import nextflow.SysEnv +import nextflow.file.FileMutex +import nextflow.util.CacheHelper +import nextflow.util.Duration +import nextflow.util.Escape +import nextflow.util.TestOnly +/** + * Handle Conda environment creation and caching + * + * @author Paolo Di Tommaso + */ +@Slf4j +@CompileStatic +class CondaCache { + static final private Object condaLock = new Object() + + /** + * Cache the prefix path for each Conda environment + */ + static final private Map> condaPrefixPaths = new ConcurrentHashMap<>() + + /** + * The Conda settings defined in the nextflow config file + */ + private CondaConfig config + + /** + * Timeout after which the environment creation is aborted + */ + private Duration createTimeout + + private String createOptions + + private boolean useMamba + + private boolean useMicromamba + + private Path configCacheDir0 + + private List channels = Collections.emptyList() + + @PackageScope String getCreateOptions() { createOptions } + + @PackageScope Duration getCreateTimeout() { createTimeout } + + @PackageScope Map getEnv() { SysEnv.get() } + + @PackageScope Path getConfigCacheDir0() { configCacheDir0 } + + @PackageScope List getChannels() { channels } + + @PackageScope String getBinaryName() { + if (useMamba) + return "mamba" + if (useMicromamba) + return "micromamba" + return "conda" + } + + @TestOnly + protected CondaCache() {} + + /** + * Create a Conda env cache object + * + * @param config A {@link Map} object + */ + CondaCache(CondaConfig config) { + this.config = config + + if( config.createTimeout() ) + createTimeout = config.createTimeout() + + if( config.createOptions() ) + createOptions = config.createOptions() + + if( config.cacheDir() ) + configCacheDir0 = config.cacheDir().toAbsolutePath() + + if( config.useMamba() && config.useMicromamba() ) + throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one") + + if( config.useMamba() ) { + useMamba = config.useMamba() + } + + if( config.useMicromamba() ) + useMicromamba = config.useMicromamba() + + if( config.getChannels() ) + channels = config.getChannels() + } + + /** + * Retrieve the directory where store the conda environment. + * + * If tries these setting in the following order: + * 1) {@code conda.cacheDir} setting in the nextflow config file; + * 2) the {@code $workDir/conda} path + * + * @return + * the {@code Path} where store the conda envs + */ + @PackageScope + Path getCacheDir() { + + def cacheDir = configCacheDir0 + + if( !cacheDir && getEnv().NXF_CONDA_CACHEDIR ) + cacheDir = getEnv().NXF_CONDA_CACHEDIR as Path + + if( !cacheDir ) + cacheDir = getSessionWorkDir().resolve('conda') + + if( cacheDir.fileSystem != FileSystems.default ) { + throw new IOException("Cannot store Conda environments to a remote work directory -- Use a POSIX compatible work directory or specify an alternative path with the `conda.cacheDir` config setting") + } + + if( !cacheDir.exists() && !cacheDir.mkdirs() ) { + throw new IOException("Failed to create Conda cache directory: $cacheDir -- Make sure a file with the same name does not exist and you have write permission") + } + + return cacheDir + } + + @PackageScope Path getSessionWorkDir() { + Global.session.workDir + } + + @PackageScope + boolean isYamlFilePath(String str) { + (str.endsWith('.yml') || str.endsWith('.yaml')) && !str.contains('\n') + } + + boolean isTextFilePath(String str) { + str.endsWith('.txt') && !str.contains('\n') + } + + /** + * Get the path on the file system where store a Conda environment + * + * @param condaEnv The conda environment + * @return the conda unique prefix {@link Path} where the env is created + */ + @PackageScope + Path condaPrefixPath(String condaEnv) { + assert condaEnv + + String content + String name = 'env' + // check if it's a remote uri + if( isYamlUriPath(condaEnv) ) { + content = condaEnv + } + // check if it's a YAML file + else if( isYamlFilePath(condaEnv) ) { + try { + final path = condaEnv as Path + content = path.text + + } + catch( NoSuchFileException e ) { + throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") + } + catch( Exception e ) { + throw new IllegalArgumentException("Error parsing Conda environment YAML file: $condaEnv -- Check the log file for details", e) + } + } + else if( isTextFilePath(condaEnv) ) { + try { + final path = condaEnv as Path + content = path.text + } + catch( NoSuchFileException e ) { + throw new IllegalArgumentException("Conda environment file does not exist: $condaEnv") + } + catch( Exception e ) { + throw new IllegalArgumentException("Error parsing Conda environment text file: $condaEnv -- Check the log file for details", e) + } + } + // it's interpreted as user provided prefix directory + else if( condaEnv.contains('/') ) { + final prefix = condaEnv as Path + if( !prefix.isDirectory() ) + throw new IllegalArgumentException("Conda prefix path does not exist or is not a directory: $prefix") + if( prefix.fileSystem != FileSystems.default ) + throw new IllegalArgumentException("Conda prefix path must be a POSIX file path: $prefix") + + return prefix + } + else if( condaEnv.contains('\n') ) { + throw new IllegalArgumentException("Invalid Conda environment definition: $condaEnv") + } + else { + content = condaEnv + } + + final hash = CacheHelper.hasher(content).hash().toString() + getCacheDir().resolve("$name-$hash") + } + + /** + * Run the conda tool to create an environment in the file system. + * + * @param condaEnv The conda environment definition + * @return the conda environment prefix {@link Path} + */ + @PackageScope + Path createLocalCondaEnv(String condaEnv, Path prefixPath) { + + if( prefixPath.isDirectory() ) { + log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath" + return prefixPath + } + + final file = new File("${prefixPath.parent}/.${prefixPath.name}.lock") + final wait = "Another Nextflow instance is creating the conda environment $condaEnv -- please wait till it completes" + final err = "Unable to acquire exclusive lock after $createTimeout on file: $file" + + final mutex = new FileMutex(target: file, timeout: createTimeout, waitMessage: wait, errorMessage: err) + try { + mutex .lock { createLocalCondaEnv0(condaEnv, prefixPath) } + } + finally { + file.delete() + } + + return prefixPath + } + + @PackageScope + Path makeAbsolute( String envFile ) { + Paths.get(envFile).toAbsolutePath() + } + + @PackageScope boolean isYamlUriPath(String env) { + env.startsWith('http://') || env.startsWith('https://') + } + + @PackageScope + Path createLocalCondaEnv0(String condaEnv, Path prefixPath) { + if( prefixPath.isDirectory() ) { + log.debug "${binaryName} found local env for environment=$condaEnv; path=$prefixPath" + return prefixPath + } + + log.info "Creating env using ${binaryName}: $condaEnv [cache $prefixPath]" + + String opts = createOptions ? "$createOptions " : '' + + def cmd + if( isYamlFilePath(condaEnv) ) { + final target = isYamlUriPath(condaEnv) ? condaEnv : Escape.path(makeAbsolute(condaEnv)) + final yesOpt = binaryName=="mamba" || binaryName == "micromamba" ? '--yes ' : '' + cmd = "${binaryName} env create ${yesOpt}--prefix ${Escape.path(prefixPath)} --file ${target}" + } + else if( isTextFilePath(condaEnv) ) { + cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} --file ${Escape.path(makeAbsolute(condaEnv))}" + } + + else { + final channelsOpt = channels.collect(it -> "-c $it ").join('') + cmd = "${binaryName} create ${opts}--yes --quiet --prefix ${Escape.path(prefixPath)} ${channelsOpt}$condaEnv" + } + + try { + // Parallel execution of conda causes data and package corruption. + // https://github.com/nextflow-io/nextflow/issues/4233 + // https://github.com/conda/conda/issues/13037 + // Should be removed as soon as the upstream bug is fixed and released. + synchronized(condaLock) { + runCommand( cmd ) + } + log.debug "'${binaryName}' create complete env=$condaEnv path=$prefixPath" + } + catch( Exception e ){ + // clean-up to avoid to keep eventually corrupted image file + prefixPath.delete() + throw e + } + return prefixPath + } + + @PackageScope + int runCommand( String cmd ) { + log.trace """${binaryName} create + command: $cmd + timeout: $createTimeout""".stripIndent(true) + + final max = createTimeout.toMillis() + final builder = new ProcessBuilder(['bash','-c',cmd]) + final proc = builder.redirectErrorStream(true).start() + final err = new StringBuilder() + final consumer = proc.consumeProcessOutputStream(err) + proc.waitForOrKill(max) + def status = proc.exitValue() + if( status != 0 ) { + consumer.join() + def msg = "Failed to create Conda environment\n command: $cmd\n status : $status\n message:\n" + msg += err.toString().trim().indent(' ') + throw new IllegalStateException(msg) + } + return status + } + + /** + * Given a remote image URL returns a {@link DataflowVariable} which holds + * the local image path. + * + * This method synchronise multiple concurrent requests so that only one + * image download is actually executed. + * + * @param condaEnv + * Conda environment string + * @return + * The {@link DataflowVariable} which hold (and pull) the local image file + */ + @PackageScope + DataflowVariable getLazyImagePath(String condaEnv) { + final prefixPath = condaPrefixPath(condaEnv) + final condaEnvPath = prefixPath.toString() + if( condaEnvPath in condaPrefixPaths ) { + log.trace "${binaryName} found local environment `$condaEnv`" + return condaPrefixPaths[condaEnvPath] + } + + synchronized (condaPrefixPaths) { + def result = condaPrefixPaths[condaEnvPath] + if( result == null ) { + result = new LazyDataflowVariable({ createLocalCondaEnv(condaEnv, prefixPath) }) + condaPrefixPaths[condaEnvPath] = result + } + else { + log.trace "${binaryName} found local cache for environment `$condaEnv` (2)" + } + return result + } + } + + /** + * Create a conda environment caching it in the file system. + * + * This method synchronise multiple concurrent requests so that only one + * environment is actually created. + * + * @param condaEnv The conda environment string + * @return the local environment path prefix {@link Path} + */ + Path getCachePathFor(String condaEnv) { + def promise = getLazyImagePath(condaEnv) + def result = promise.getVal() + if( promise.isError() ) + throw new IllegalStateException(promise.getError()) + if( !result ) + throw new IllegalStateException("Cannot create Conda environment `$condaEnv`") + log.trace "Conda cache for env `$condaEnv` path=$result" + return result + } + +} diff --git a/plugins/nf-conda/src/main/nextflow/conda/CondaConfig.groovy b/plugins/nf-conda/src/main/nextflow/conda/CondaConfig.groovy new file mode 100644 index 0000000000..89af2f10cb --- /dev/null +++ b/plugins/nf-conda/src/main/nextflow/conda/CondaConfig.groovy @@ -0,0 +1,129 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.conda + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import nextflow.config.schema.ConfigOption +import nextflow.config.schema.ConfigScope +import nextflow.config.schema.ScopeName +import nextflow.script.dsl.Description +import nextflow.util.Duration + +/** + * Model Conda configuration + * + * @author Paolo Di Tommaso + */ +@ScopeName("conda") +@Description(""" + The `conda` scope controls the creation of Conda environments by the Conda package manager. +""") +@CompileStatic +class CondaConfig implements ConfigScope { + + @ConfigOption + @Description(""" + Execute tasks with Conda environments (default: `false`). + """) + final boolean enabled + + @ConfigOption + @Description(""" + The path where Conda environments are stored. It should be accessible from all compute nodes when using a shared file system. + """) + final String cacheDir + + @ConfigOption + @Description(""" + The list of Conda channels that can be used to resolve Conda packages. Channel priority decreases from left to right. + """) + final List channels + + @ConfigOption + @Description(""" + Extra command line options for the `conda create` command. See the [Conda documentation](https://docs.conda.io/projects/conda/en/latest/commands/create.html) for more information. + """) + final String createOptions + + @ConfigOption + @Description(""" + The amount of time to wait for the Conda environment to be created before failing (default: `20 min`). + """) + final Duration createTimeout + + @ConfigOption + @Description(""" + Use [Mamba](https://github.com/mamba-org/mamba) instead of `conda` to create Conda environments (default: `false`). + """) + final boolean useMamba + + @ConfigOption + @Description(""" + Use [Micromamba](https://mamba.readthedocs.io/en/latest/user_guide/micromamba.html) instead of `conda` to create Conda environments (default: `false`). + """) + final boolean useMicromamba + + /* required by extension point -- do not remove */ + CondaConfig() {} + + CondaConfig(Map opts, Map env) { + enabled = opts.enabled != null + ? opts.enabled as boolean + : (env.NXF_CONDA_ENABLED?.toString() == 'true') + cacheDir = opts.cacheDir + channels = parseChannels(opts.channels) + createOptions = opts.createOptions + createTimeout = opts.createTimeout as Duration ?: Duration.of('20min') + useMamba = opts.useMamba as boolean + useMicromamba = opts.useMicromamba as boolean + + if( useMamba && useMicromamba ) + throw new IllegalArgumentException("Both conda.useMamba and conda.useMicromamba were enabled -- Please choose only one") + } + + private List parseChannels(Object value) { + if( !value ) + return Collections.emptyList() + if( value instanceof List ) + return value + if( value instanceof CharSequence ) + return value.tokenize(',').collect(it -> it.trim()) + throw new IllegalArgumentException("Unexpected conda.channels value: $value") + } + + Duration createTimeout() { + createTimeout + } + + String createOptions() { + createOptions + } + + Path cacheDir() { + cacheDir as Path + } + + boolean useMamba() { + useMamba + } + + boolean useMicromamba() { + useMicromamba + } +} diff --git a/plugins/nf-conda/src/main/nextflow/conda/CondaPackageProvider.groovy b/plugins/nf-conda/src/main/nextflow/conda/CondaPackageProvider.groovy new file mode 100644 index 0000000000..0d347f0052 --- /dev/null +++ b/plugins/nf-conda/src/main/nextflow/conda/CondaPackageProvider.groovy @@ -0,0 +1,102 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.conda + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.packages.PackageProvider +import nextflow.packages.PackageSpec +import nextflow.util.Escape + +/** + * Conda package provider implementation + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class CondaPackageProvider implements PackageProvider { + + private final CondaCache cache + private final CondaConfig config + + CondaPackageProvider(CondaConfig config) { + this.config = config + this.cache = new CondaCache(config) + } + + @Override + String getName() { + return 'conda' + } + + @Override + boolean isAvailable() { + try { + def process = new ProcessBuilder('conda', '--version').start() + process.waitFor() + return process.exitValue() == 0 + } catch (Exception e) { + log.debug "Conda not available: ${e.message}" + return false + } + } + + @Override + Path createEnvironment(PackageSpec spec) { + if (!supportsSpec(spec)) { + throw new IllegalArgumentException("Unsupported package spec for conda: ${spec}") + } + + String condaEnv + if (spec.hasEnvironmentFile()) { + // Handle environment file + condaEnv = spec.environment + } else if (spec.hasEntries()) { + // Handle package list + condaEnv = spec.entries.join(' ') + } else { + throw new IllegalArgumentException("Package spec must have either environment file or entries") + } + + return cache.getCachePathFor(condaEnv) + } + + @Override + String getActivationScript(Path envPath) { + def binaryName = cache.getBinaryName() + final command = config.useMicromamba() + ? 'eval "$(micromamba shell hook --shell bash)" && micromamba activate' + : 'source $(conda info --json | awk \'/conda_prefix/ { gsub(/"|,/, "", $2); print $2 }\')/bin/activate' + + return """\ + ${command} ${Escape.path(envPath)} + """.stripIndent() + } + + @Override + boolean supportsSpec(PackageSpec spec) { + return spec.provider == getName() + } + + @Override + Object getConfig() { + return config + } +} \ No newline at end of file diff --git a/plugins/nf-conda/src/main/nextflow/conda/CondaPlugin.groovy b/plugins/nf-conda/src/main/nextflow/conda/CondaPlugin.groovy new file mode 100644 index 0000000000..3c9b514122 --- /dev/null +++ b/plugins/nf-conda/src/main/nextflow/conda/CondaPlugin.groovy @@ -0,0 +1,49 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.conda + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.packages.PackageManager +import nextflow.plugin.BasePlugin +import org.pf4j.PluginWrapper + +/** + * Nextflow Conda Package Manager Plugin + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class CondaPlugin extends BasePlugin { + + CondaPlugin(PluginWrapper wrapper) { + super(wrapper) + } + + @Override + void start() { + log.info "Starting Conda package manager plugin" + super.start() + } + + @Override + void stop() { + log.info "Stopping Conda package manager plugin" + super.stop() + } +} \ No newline at end of file diff --git a/plugins/nf-conda/src/main/nextflow/conda/CondaProviderExtension.groovy b/plugins/nf-conda/src/main/nextflow/conda/CondaProviderExtension.groovy new file mode 100644 index 0000000000..bc8693a813 --- /dev/null +++ b/plugins/nf-conda/src/main/nextflow/conda/CondaProviderExtension.groovy @@ -0,0 +1,42 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.conda + +import groovy.transform.CompileStatic +import nextflow.ISession +import nextflow.packages.PackageProvider +import nextflow.packages.PackageProviderExtension + +/** + * Conda package provider extension + * + * @author Edmund Miller + */ +@CompileStatic +class CondaProviderExtension implements PackageProviderExtension { + + @Override + PackageProvider createProvider(ISession session) { + def condaConfig = new CondaConfig(session.config.navigate('conda') as Map ?: [:], System.getenv()) + return new CondaPackageProvider(condaConfig) + } + + @Override + int getPriority() { + return 100 + } +} \ No newline at end of file diff --git a/plugins/nf-conda/src/main/resources/META-INF/MANIFEST.MF b/plugins/nf-conda/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..988ea44b0e --- /dev/null +++ b/plugins/nf-conda/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,8 @@ +Plugin-Class: nextflow.conda.CondaPlugin +Plugin-Id: nf-conda +Plugin-Version: 1.0.0 +Plugin-Provider: Seqera Labs +Plugin-Description: Conda package manager support for Nextflow +Plugin-License: Apache-2.0 +Plugin-Requires: >=25.04.0 +Nextflow-Version: >=25.04.0 \ No newline at end of file diff --git a/plugins/nf-conda/src/main/resources/META-INF/extensions.idx b/plugins/nf-conda/src/main/resources/META-INF/extensions.idx new file mode 100644 index 0000000000..8e6ce54edb --- /dev/null +++ b/plugins/nf-conda/src/main/resources/META-INF/extensions.idx @@ -0,0 +1 @@ +nextflow.conda.CondaProviderExtension \ No newline at end of file diff --git a/plugins/nf-conda/src/resources/META-INF/MANIFEST.MF b/plugins/nf-conda/src/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..b97747010d --- /dev/null +++ b/plugins/nf-conda/src/resources/META-INF/MANIFEST.MF @@ -0,0 +1,6 @@ +Manifest-Version: 1.0 +Plugin-Class: nextflow.conda.CondaPlugin +Plugin-Id: nf-conda +Plugin-Version: 1.0.0 +Plugin-Provider: Seqera Labs +Plugin-Requires: >=25.04.0 \ No newline at end of file diff --git a/plugins/nf-pixi/build.gradle b/plugins/nf-pixi/build.gradle new file mode 100644 index 0000000000..556189151d --- /dev/null +++ b/plugins/nf-pixi/build.gradle @@ -0,0 +1,45 @@ +/* + * Copyright 2021-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +apply plugin: 'java' +apply plugin: 'java-test-fixtures' +apply plugin: 'idea' +apply plugin: 'groovy' + +sourceSets { + main.java.srcDirs = [] + main.groovy.srcDirs = ['src/main'] + main.resources.srcDirs = ['src/resources'] + test.groovy.srcDirs = ['src/test'] + test.java.srcDirs = [] + test.resources.srcDirs = ['src/testResources'] +} + +configurations { + // see https://docs.gradle.org/4.1/userguide/dependency_management.html#sub:exclude_transitive_dependencies + runtimeClasspath.exclude group: 'org.slf4j', module: 'slf4j-api' +} + +dependencies { + compileOnly project(':nextflow') + compileOnly 'org.slf4j:slf4j-api:2.0.17' + compileOnly 'org.pf4j:pf4j:3.12.0' + + testImplementation(testFixtures(project(":nextflow"))) + testImplementation project(':nextflow') + testImplementation "org.apache.groovy:groovy:4.0.28" + testImplementation "org.apache.groovy:groovy-nio:4.0.28" +} \ No newline at end of file diff --git a/plugins/nf-pixi/src/main/nextflow/pixi/PixiCache.groovy b/plugins/nf-pixi/src/main/nextflow/pixi/PixiCache.groovy new file mode 100644 index 0000000000..e9aa316e1d --- /dev/null +++ b/plugins/nf-pixi/src/main/nextflow/pixi/PixiCache.groovy @@ -0,0 +1,367 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.pixi + +import java.nio.file.FileSystems +import java.nio.file.NoSuchFileException +import java.nio.file.Path +import java.nio.file.Paths +import java.util.concurrent.ConcurrentHashMap + +import groovy.transform.CompileStatic +import groovy.transform.PackageScope +import groovy.util.logging.Slf4j +import groovyx.gpars.dataflow.DataflowVariable +import groovyx.gpars.dataflow.LazyDataflowVariable +import nextflow.Global +import nextflow.file.FileMutex +import nextflow.util.CacheHelper +import nextflow.util.Duration +import nextflow.util.Escape +import nextflow.util.TestOnly + +/** + * Handle Pixi environment creation and caching + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class PixiCache { + + /** + * Cache the prefix path for each Pixi environment + */ + static final private Map> pixiPrefixPaths = new ConcurrentHashMap<>() + + /** + * The Pixi settings defined in the nextflow config file + */ + private PixiConfig config + + /** + * Timeout after which the environment creation is aborted + */ + private Duration createTimeout = Duration.of('20min') + + private String createOptions + + private Path configCacheDir0 + + @PackageScope String getCreateOptions() { createOptions } + + @PackageScope Duration getCreateTimeout() { createTimeout } + + @PackageScope Map getEnv() { System.getenv() } + + @PackageScope Path getConfigCacheDir0() { configCacheDir0 } + + @TestOnly + protected PixiCache() {} + + /** + * Create a Pixi env cache object + * + * @param config A {@link PixiConfig} object + */ + PixiCache(PixiConfig config) { + this.config = config + + if( config.createTimeout() ) + createTimeout = config.createTimeout() + + if( config.createOptions() ) + createOptions = config.createOptions() + + if( config.cacheDir() ) + configCacheDir0 = config.cacheDir().toAbsolutePath() + } + + /** + * Retrieve the directory where store the pixi environment. + * + * If tries these setting in the following order: + * 1) {@code pixi.cacheDir} setting in the nextflow config file; + * 2) the {@code $workDir/pixi} path + * + * @return + * the {@code Path} where store the pixi envs + */ + @PackageScope + Path getCacheDir() { + + def cacheDir = configCacheDir0 + + if( !cacheDir && getEnv().NXF_PIXI_CACHEDIR ) + cacheDir = getEnv().NXF_PIXI_CACHEDIR as Path + + if( !cacheDir ) + cacheDir = getSessionWorkDir().resolve('pixi') + + if( cacheDir.fileSystem != FileSystems.default ) { + throw new IOException("Cannot store Pixi environments to a remote work directory -- Use a POSIX compatible work directory or specify an alternative path with the `pixi.cacheDir` config setting") + } + + if( !cacheDir.exists() && !cacheDir.mkdirs() ) { + throw new IOException("Failed to create Pixi cache directory: $cacheDir -- Make sure a file with the same name does not exist and you have write permission") + } + + return cacheDir + } + + @PackageScope Path getSessionWorkDir() { + Global.session.workDir + } + + @PackageScope + boolean isTomlFilePath(String str) { + str.endsWith('.toml') && !str.contains('\n') + } + + @PackageScope + boolean isLockFilePath(String str) { + str.endsWith('.lock') && !str.contains('\n') + } + + /** + * Get the path on the file system where store a Pixi environment + * + * @param pixiEnv The pixi environment + * @return the pixi unique prefix {@link Path} where the env is created + */ + @PackageScope + Path pixiPrefixPath(String pixiEnv) { + assert pixiEnv + + String content + String name = 'env' + + // check if it's a TOML file (pixi.toml or pyproject.toml) + if( isTomlFilePath(pixiEnv) ) { + try { + final path = pixiEnv as Path + content = path.text + name = path.baseName + } + catch( NoSuchFileException e ) { + throw new IllegalArgumentException("Pixi environment file does not exist: $pixiEnv") + } + catch( Exception e ) { + throw new IllegalArgumentException("Error parsing Pixi environment TOML file: $pixiEnv -- Check the log file for details", e) + } + } + // check if it's a lock file (pixi.lock) + else if( isLockFilePath(pixiEnv) ) { + try { + final path = pixiEnv as Path + content = path.text + name = path.baseName + } + catch( NoSuchFileException e ) { + throw new IllegalArgumentException("Pixi lock file does not exist: $pixiEnv") + } + catch( Exception e ) { + throw new IllegalArgumentException("Error parsing Pixi lock file: $pixiEnv -- Check the log file for details", e) + } + } + // it's interpreted as user provided prefix directory + else if( pixiEnv.contains('/') ) { + final prefix = pixiEnv as Path + if( !prefix.isDirectory() ) + throw new IllegalArgumentException("Pixi prefix path does not exist or is not a directory: $prefix") + if( prefix.fileSystem != FileSystems.default ) + throw new IllegalArgumentException("Pixi prefix path must be a POSIX file path: $prefix") + + return prefix + } + else if( pixiEnv.contains('\n') ) { + throw new IllegalArgumentException("Invalid Pixi environment definition: $pixiEnv") + } + else { + // it's interpreted as a package specification + content = pixiEnv + } + + final hash = CacheHelper.hasher(content).hash().toString() + getCacheDir().resolve("$name-$hash") + } + + /** + * Run the pixi tool to create an environment in the file system. + * + * @param pixiEnv The pixi environment definition + * @return the pixi environment prefix {@link Path} + */ + @PackageScope + Path createLocalPixiEnv(String pixiEnv, Path prefixPath) { + + if( prefixPath.isDirectory() ) { + log.debug "pixi found local env for environment=$pixiEnv; path=$prefixPath" + return prefixPath + } + + final file = new File("${prefixPath.parent}/.${prefixPath.name}.lock") + final wait = "Another Nextflow instance is creating the pixi environment $pixiEnv -- please wait till it completes" + final err = "Unable to acquire exclusive lock after $createTimeout on file: $file" + + final mutex = new FileMutex(target: file, timeout: createTimeout, waitMessage: wait, errorMessage: err) + try { + mutex .lock { createLocalPixiEnv0(pixiEnv, prefixPath) } + } + finally { + file.delete() + } + + return prefixPath + } + + @PackageScope + Path makeAbsolute( String envFile ) { + Paths.get(envFile).toAbsolutePath() + } + + @PackageScope + Path createLocalPixiEnv0(String pixiEnv, Path prefixPath) { + log.info "Creating env using pixi: $pixiEnv [cache $prefixPath]" + + String opts = createOptions ? "$createOptions " : '' + + def cmd + if( isTomlFilePath(pixiEnv) || isLockFilePath(pixiEnv) ) { + final target = Escape.path(makeAbsolute(pixiEnv)) + final projectDir = makeAbsolute(pixiEnv).parent + + // Create environment from project file + cmd = "cd ${Escape.path(projectDir)} && pixi install ${opts}" + + // Set up the environment directory + prefixPath.mkdirs() + final envLink = prefixPath.resolve('.pixi') + if( !envLink.exists() ) { + envLink.toFile().createNewFile() + envLink.write(projectDir.toString()) + } + } + else { + // Create environment from package specification + prefixPath.mkdirs() + final manifestFile = prefixPath.resolve('pixi.toml') + + // Create a simple pixi.toml with the requested packages + manifestFile.text = """\ +[project] +name = "nextflow-env" +version = "0.1.0" +description = "Nextflow generated Pixi environment" +channels = ["conda-forge"] + +[dependencies] +${pixiEnv} +""".stripIndent() + + cmd = "cd ${Escape.path(prefixPath)} && pixi install ${opts}" + } + + try { + runCommand( cmd ) + log.debug "'pixi' create complete env=$pixiEnv path=$prefixPath" + } + catch( Exception e ){ + // clean-up to avoid to keep eventually corrupted image file + prefixPath.delete() + throw e + } + return prefixPath + } + + @PackageScope + int runCommand( String cmd ) { + log.trace """pixi create + command: $cmd + timeout: $createTimeout""".stripIndent(true) + + final max = createTimeout.toMillis() + final builder = new ProcessBuilder(['bash','-c',cmd]) + final proc = builder.redirectErrorStream(true).start() + final err = new StringBuilder() + final consumer = proc.consumeProcessOutputStream(err) + proc.waitForOrKill(max) + def status = proc.exitValue() + if( status != 0 ) { + consumer.join() + def msg = "Failed to create Pixi environment\n command: $cmd\n status : $status\n message:\n" + msg += err.toString().trim().indent(' ') + throw new IllegalStateException(msg) + } + return status + } + + /** + * Given a remote image URL returns a {@link DataflowVariable} which holds + * the local image path. + * + * This method synchronise multiple concurrent requests so that only one + * image download is actually executed. + * + * @param pixiEnv + * Pixi environment string + * @return + * The {@link DataflowVariable} which hold (and pull) the local image file + */ + @PackageScope + DataflowVariable getLazyImagePath(String pixiEnv) { + final prefixPath = pixiPrefixPath(pixiEnv) + final pixiEnvPath = prefixPath.toString() + if( pixiEnvPath in pixiPrefixPaths ) { + log.trace "pixi found local environment `$pixiEnv`" + return pixiPrefixPaths[pixiEnvPath] + } + + synchronized (pixiPrefixPaths) { + def result = pixiPrefixPaths[pixiEnvPath] + if( result == null ) { + result = new LazyDataflowVariable({ createLocalPixiEnv(pixiEnv, prefixPath) }) + pixiPrefixPaths[pixiEnvPath] = result + } + else { + log.trace "pixi found local cache for environment `$pixiEnv` (2)" + } + return result + } + } + + /** + * Create a pixi environment caching it in the file system. + * + * This method synchronise multiple concurrent requests so that only one + * environment is actually created. + * + * @param pixiEnv The pixi environment string + * @return the local environment path prefix {@link Path} + */ + Path getCachePathFor(String pixiEnv) { + def promise = getLazyImagePath(pixiEnv) + def result = promise.getVal() + if( promise.isError() ) + throw new IllegalStateException(promise.getError()) + if( !result ) + throw new IllegalStateException("Cannot create Pixi environment `$pixiEnv`") + log.trace "Pixi cache for env `$pixiEnv` path=$result" + return result + } + +} diff --git a/plugins/nf-pixi/src/main/nextflow/pixi/PixiConfig.groovy b/plugins/nf-pixi/src/main/nextflow/pixi/PixiConfig.groovy new file mode 100644 index 0000000000..968d638556 --- /dev/null +++ b/plugins/nf-pixi/src/main/nextflow/pixi/PixiConfig.groovy @@ -0,0 +1,60 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.pixi + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import nextflow.util.Duration + +/** + * Model Pixi configuration + * + * @author Edmund Miller + */ +@CompileStatic +class PixiConfig extends LinkedHashMap { + + private Map env + + /* required by Kryo deserialization -- do not remove */ + private PixiConfig() { } + + PixiConfig(Map config, Map env) { + super(config) + this.env = env + } + + boolean isEnabled() { + def enabled = get('enabled') + if( enabled == null ) + enabled = env.get('NXF_PIXI_ENABLED') + return enabled?.toString() == 'true' + } + + Duration createTimeout() { + get('createTimeout') as Duration + } + + String createOptions() { + get('createOptions') as String + } + + Path cacheDir() { + get('cacheDir') as Path + } +} diff --git a/plugins/nf-pixi/src/main/nextflow/pixi/PixiPackageProvider.groovy b/plugins/nf-pixi/src/main/nextflow/pixi/PixiPackageProvider.groovy new file mode 100644 index 0000000000..2975ebb2f0 --- /dev/null +++ b/plugins/nf-pixi/src/main/nextflow/pixi/PixiPackageProvider.groovy @@ -0,0 +1,112 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.pixi + +import java.nio.file.Path + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.packages.PackageProvider +import nextflow.packages.PackageSpec +import nextflow.util.Escape + +/** + * Pixi package provider implementation + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class PixiPackageProvider implements PackageProvider { + + private final PixiCache cache + private final PixiConfig config + + PixiPackageProvider(PixiConfig config) { + this.config = config + this.cache = new PixiCache(config) + } + + @Override + String getName() { + return 'pixi' + } + + @Override + boolean isAvailable() { + try { + def process = new ProcessBuilder('pixi', '--version').start() + process.waitFor() + return process.exitValue() == 0 + } catch (Exception e) { + log.debug "Pixi not available: ${e.message}" + return false + } + } + + @Override + Path createEnvironment(PackageSpec spec) { + if (!supportsSpec(spec)) { + throw new IllegalArgumentException("Unsupported package spec for pixi: ${spec}") + } + + String pixiEnv + if (spec.hasEnvironmentFile()) { + // Handle environment file + pixiEnv = spec.environment + } else if (spec.hasEntries()) { + // Handle package list + pixiEnv = spec.entries.join(' ') + } else { + throw new IllegalArgumentException("Package spec must have either environment file or entries") + } + + return cache.getCachePathFor(pixiEnv) + } + + @Override + String getActivationScript(Path envPath) { + def result = "" + + // Check if there's a .pixi file that points to the project directory + final pixiFile = envPath.resolve('.pixi') + if (pixiFile.exists()) { + // Read the project directory path + final projectDir = pixiFile.text.trim() + result += "cd ${Escape.path(projectDir as String)} && " + result += "eval \"\$(pixi shell-hook --shell bash)\" && " + result += "cd \"\$OLDPWD\"\n" + } else { + // Direct activation from environment directory + result += "cd ${Escape.path(envPath)} && " + result += "eval \"\$(pixi shell-hook --shell bash)\" && " + result += "cd \"\$OLDPWD\"\n" + } + + return result + } + + @Override + boolean supportsSpec(PackageSpec spec) { + return spec.provider == getName() + } + + @Override + Object getConfig() { + return config + } +} \ No newline at end of file diff --git a/plugins/nf-pixi/src/main/nextflow/pixi/PixiPlugin.groovy b/plugins/nf-pixi/src/main/nextflow/pixi/PixiPlugin.groovy new file mode 100644 index 0000000000..1d1f131440 --- /dev/null +++ b/plugins/nf-pixi/src/main/nextflow/pixi/PixiPlugin.groovy @@ -0,0 +1,48 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.pixi + +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.plugin.BasePlugin +import org.pf4j.PluginWrapper + +/** + * Nextflow Pixi Package Manager Plugin + * + * @author Edmund Miller + */ +@Slf4j +@CompileStatic +class PixiPlugin extends BasePlugin { + + PixiPlugin(PluginWrapper wrapper) { + super(wrapper) + } + + @Override + void start() { + log.info "Starting Pixi package manager plugin" + super.start() + } + + @Override + void stop() { + log.info "Stopping Pixi package manager plugin" + super.stop() + } +} \ No newline at end of file diff --git a/plugins/nf-pixi/src/main/nextflow/pixi/PixiProviderExtension.groovy b/plugins/nf-pixi/src/main/nextflow/pixi/PixiProviderExtension.groovy new file mode 100644 index 0000000000..95b2e484ce --- /dev/null +++ b/plugins/nf-pixi/src/main/nextflow/pixi/PixiProviderExtension.groovy @@ -0,0 +1,42 @@ +/* + * Copyright 2013-2024, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.pixi + +import groovy.transform.CompileStatic +import nextflow.ISession +import nextflow.packages.PackageProvider +import nextflow.packages.PackageProviderExtension + +/** + * Pixi package provider extension + * + * @author Edmund Miller + */ +@CompileStatic +class PixiProviderExtension implements PackageProviderExtension { + + @Override + PackageProvider createProvider(ISession session) { + def pixiConfig = new PixiConfig(session.config.navigate('pixi') as Map ?: [:], System.getenv()) + return new PixiPackageProvider(pixiConfig) + } + + @Override + int getPriority() { + return 100 + } +} \ No newline at end of file diff --git a/plugins/nf-pixi/src/main/resources/META-INF/MANIFEST.MF b/plugins/nf-pixi/src/main/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..863ff23ef2 --- /dev/null +++ b/plugins/nf-pixi/src/main/resources/META-INF/MANIFEST.MF @@ -0,0 +1,8 @@ +Plugin-Class: nextflow.pixi.PixiPlugin +Plugin-Id: nf-pixi +Plugin-Version: 1.0.0 +Plugin-Provider: Seqera Labs +Plugin-Description: Pixi package manager support for Nextflow +Plugin-License: Apache-2.0 +Plugin-Requires: >=25.04.0 +Nextflow-Version: >=25.04.0 \ No newline at end of file diff --git a/plugins/nf-pixi/src/main/resources/META-INF/extensions.idx b/plugins/nf-pixi/src/main/resources/META-INF/extensions.idx new file mode 100644 index 0000000000..7ef30aa06c --- /dev/null +++ b/plugins/nf-pixi/src/main/resources/META-INF/extensions.idx @@ -0,0 +1 @@ +nextflow.pixi.PixiProviderExtension \ No newline at end of file diff --git a/plugins/nf-pixi/src/resources/META-INF/MANIFEST.MF b/plugins/nf-pixi/src/resources/META-INF/MANIFEST.MF new file mode 100644 index 0000000000..be508d4bc2 --- /dev/null +++ b/plugins/nf-pixi/src/resources/META-INF/MANIFEST.MF @@ -0,0 +1,6 @@ +Manifest-Version: 1.0 +Plugin-Class: nextflow.pixi.PixiPlugin +Plugin-Id: nf-pixi +Plugin-Version: 1.0.0 +Plugin-Provider: Seqera Labs +Plugin-Requires: >=25.04.0 \ No newline at end of file diff --git a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy index dea72f4f5c..0c97caed2f 100644 --- a/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy +++ b/plugins/nf-wave/src/main/io/seqera/wave/plugin/WaveClient.groovy @@ -67,6 +67,10 @@ import nextflow.util.SysHelper import nextflow.util.Threads import org.slf4j.Logger import org.slf4j.LoggerFactory +import static nextflow.util.SysHelper.DEFAULT_DOCKER_PLATFORM +import nextflow.packages.PackageManager +import nextflow.packages.PackageSpec + /** * Wave client service * @@ -471,6 +475,7 @@ class WaveClient { def attrs = new HashMap() attrs.container = containerImage attrs.conda = task.config.conda as String + attrs.package = task.config.package if( bundle!=null && bundle.dockerfile ) { attrs.dockerfile = bundle.dockerfile.text } @@ -534,6 +539,30 @@ class WaveClient { } } + /* + * If 'package' directive is specified use it to create a container file + * to assemble the target container + */ + if( attrs.package && !packagesSpec ) { + if( containerScript ) + throw new IllegalArgumentException("Unexpected package and $scriptType conflict while resolving wave container") + + // Check if new package system is enabled + if( PackageManager.isEnabled(session) ) { + try { + def defaultProvider = session.config.navigate('packages.provider', 'conda') as String + PackageSpec spec = PackageManager.parseSpec(attrs.package, defaultProvider) + + if( spec ) { + packagesSpec = convertToWavePackagesSpec(spec) + } + } + catch( Exception e ) { + log.warn "Failed to parse package specification for Wave: ${e.message}" + } + } + } + /* * The process should declare at least a container image name via 'container' directive * or a dockerfile file to build, otherwise there's no job to be done by wave @@ -743,4 +772,64 @@ class WaveClient { value.startsWith('http://') || value.startsWith('https://') } + protected HttpResponse httpSend(HttpRequest req) { + try { + return httpClient.sendAsString(req) + } + catch (IOException e) { + throw new IllegalStateException("Unable to connect Wave service: $endpoint") + } + } + + /** + * Convert a Nextflow PackageSpec to Wave PackagesSpec + */ + private PackagesSpec convertToWavePackagesSpec(nextflow.packages.PackageSpec spec) { + def waveSpec = new PackagesSpec() + + // Map provider to Wave PackagesSpec Type + def waveType = mapProviderToWaveType(spec.provider) + if (waveType) { + waveSpec.withType(waveType) + } + + // Set entries or environment + if (spec.hasEntries()) { + waveSpec.withEntries(spec.entries) + } + + if (spec.hasEnvironmentFile()) { + waveSpec.withEnvironment(spec.environment) + } + + // Set channels (conda-specific) + if (spec.channels && !spec.channels.empty) { + waveSpec.withChannels(spec.channels) + } + + // Set conda options if provider is conda + if (spec.provider == 'conda' && config.condaOpts()) { + waveSpec.withCondaOpts(config.condaOpts()) + } + + return waveSpec + } + + /** + * Map provider name to Wave PackagesSpec Type + */ + private PackagesSpec.Type mapProviderToWaveType(String provider) { + switch (provider?.toLowerCase()) { + case 'conda': + case 'mamba': + case 'micromamba': + return PackagesSpec.Type.CONDA + case 'pixi': + // Wave doesn't support pixi yet, so we'll use conda for now + return PackagesSpec.Type.CONDA + default: + log.warn "Unknown package provider for Wave: ${provider}, defaulting to CONDA" + return PackagesSpec.Type.CONDA + } + } } diff --git a/settings.gradle b/settings.gradle index 53d56ba13b..2c6e38a5ef 100644 --- a/settings.gradle +++ b/settings.gradle @@ -43,3 +43,5 @@ include 'plugins:nf-codecommit' include 'plugins:nf-wave' include 'plugins:nf-cloudcache' include 'plugins:nf-k8s' +include 'plugins:nf-conda' +include 'plugins:nf-pixi' diff --git a/tests/checks/pixi-env.nf b/tests/checks/pixi-env.nf new file mode 100644 index 0000000000..229fc09e9e --- /dev/null +++ b/tests/checks/pixi-env.nf @@ -0,0 +1,21 @@ +#!/usr/bin/env nextflow +workflow { + sayHello() | view +} + + +/* + * Test for Pixi environment support + */ + +process sayHello { + pixi 'cowpy' + + output: + stdout + + script: + """ + cowpy "hello pixi" + """ +} diff --git a/tests/integration-test.config b/tests/integration-test.config new file mode 100644 index 0000000000..651ad3f7be --- /dev/null +++ b/tests/integration-test.config @@ -0,0 +1,22 @@ +/* + * Configuration for integration test + * Tests both old and new package management systems + */ + +// Enable the new package management system +nextflow.preview.package = true + +// Configure package providers +packages { + provider = 'conda' // default provider +} + +// Keep existing conda/pixi configs for backward compatibility +conda { + enabled = true + channels = ['conda-forge', 'bioconda'] +} + +pixi { + enabled = true +} \ No newline at end of file diff --git a/tests/integration-test.nf b/tests/integration-test.nf new file mode 100644 index 0000000000..b492a07905 --- /dev/null +++ b/tests/integration-test.nf @@ -0,0 +1,81 @@ +#!/usr/bin/env nextflow + +/* + * Integration test for the unified package management system + * This test demonstrates the new package directive with backward compatibility + */ + +nextflow.enable.dsl=2 + +// Old style conda directive - should show deprecation warning when preview.package is enabled +process oldStyleConda { + conda 'samtools=1.17' + + output: + stdout + + script: + """ + echo "Old style conda: \$(samtools --version 2>/dev/null | head -1 || echo 'not available')" + """ +} + +// Old style pixi directive - should show deprecation warning when preview.package is enabled +process oldStylePixi { + pixi 'samtools' + + output: + stdout + + script: + """ + echo "Old style pixi: \$(samtools --version 2>/dev/null | head -1 || echo 'not available')" + """ +} + +// New style package directive with explicit provider +process newStyleExplicit { + package "samtools=1.17", provider: "conda" + + output: + stdout + + script: + """ + echo "New style explicit: \$(samtools --version 2>/dev/null | head -1 || echo 'not available')" + """ +} + +// New style package directive with default provider (from config) +process newStyleDefault { + package "samtools=1.17" + + output: + stdout + + script: + """ + echo "New style default: \$(samtools --version 2>/dev/null | head -1 || echo 'not available')" + """ +} + +// New style package directive with multiple packages +process newStyleMultiple { + package ["samtools=1.17", "bcftools=1.18"], provider: "conda" + + output: + stdout + + script: + """ + echo "New style multiple: samtools \$(samtools --version 2>/dev/null | head -1 | cut -d' ' -f2 || echo 'n/a'), bcftools \$(bcftools --version 2>/dev/null | head -1 | cut -d' ' -f2 || echo 'n/a')" + """ +} + +workflow { + oldStyleConda() | view + oldStylePixi() | view + newStyleExplicit() | view + newStyleDefault() | view + newStyleMultiple() | view +} \ No newline at end of file diff --git a/tests/package-test.config b/tests/package-test.config new file mode 100644 index 0000000000..c04254bb15 --- /dev/null +++ b/tests/package-test.config @@ -0,0 +1,14 @@ +/* + * Configuration file for package management test + */ + +// Enable the preview feature +nextflow.preview.package = true + +// Configure the default package provider +packages { + provider = 'conda' + conda { + channels = ['conda-forge', 'bioconda'] + } +} \ No newline at end of file diff --git a/tests/package-test.nf b/tests/package-test.nf new file mode 100644 index 0000000000..ad7aa16cf3 --- /dev/null +++ b/tests/package-test.nf @@ -0,0 +1,52 @@ +#!/usr/bin/env nextflow + +/* + * Test script for the unified package management system + */ + +nextflow.enable.dsl=2 + +// Test the new package directive with conda provider +process testConda { + package "samtools=1.17", provider: "conda" + + output: + stdout + + script: + """ + samtools --version | head -1 + """ +} + +// Test the new package directive with pixi provider +process testPixi { + package "samtools=1.17", provider: "pixi" + + output: + stdout + + script: + """ + samtools --version | head -1 + """ +} + +// Test the new package directive with default provider +process testDefault { + package "samtools=1.17" + + output: + stdout + + script: + """ + samtools --version | head -1 + """ +} + +workflow { + testConda() | view { "Conda: ${it.trim()}" } + testPixi() | view { "Pixi: ${it.trim()}" } + testDefault() | view { "Default: ${it.trim()}" } +} \ No newline at end of file