Skip to content

Add pixi support #6157

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 5 commits into
base: master
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/Session.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ import nextflow.script.ScriptFile
import nextflow.script.ScriptMeta
import nextflow.script.ScriptRunner
import nextflow.script.WorkflowMetadata
import nextflow.pixi.PixiConfig
import nextflow.spack.SpackConfig
import nextflow.trace.AnsiLogObserver
import nextflow.trace.TraceObserver
Expand Down Expand Up @@ -1194,6 +1195,12 @@ class Session implements ISession {
return new SpackConfig(cfg, getSystemEnv())
}

@Memoized
PixiConfig getPixiConfig() {
final cfg = config.pixi as Map ?: Collections.emptyMap()
return new PixiConfig(cfg, getSystemEnv())
}

/**
* Get the container engine configuration for the specified engine. If no engine is specified
* if returns the one enabled in the configuration file. If no configuration is found
Expand Down
6 changes: 6 additions & 0 deletions modules/nextflow/src/main/groovy/nextflow/cli/CmdRun.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,12 @@ class CmdRun extends CmdBase implements HubOptions {
@Parameter(names=['-without-spack'], description = 'Disable the use of Spack environments')
Boolean withoutSpack

@Parameter(names=['-with-pixi'], description = 'Use the specified Pixi environment package or file (must end with .toml suffix)')
String withPixi

@Parameter(names=['-without-pixi'], description = 'Disable the use of Pixi environments')
Boolean withoutPixi

@Parameter(names=['-offline'], description = 'Do not check for remote project updates')
boolean offline = System.getenv('NXF_OFFLINE')=='true'

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ class ConfigBuilder {

def result = []
if ( files ) {
for( String fileName : files ) {
for( String fileName : files ) {
def thisFile = currentDir.resolve(fileName)
if(!thisFile.exists()) {
throw new AbortOperationException("The specified configuration file does not exist: $thisFile -- check the name or choose another file")
Expand Down Expand Up @@ -597,6 +597,19 @@ class ConfigBuilder {
config.spack.enabled = true
}

if( cmdRun.withoutPixi && config.pixi instanceof Map ) {
// disable pixi execution
log.debug "Disabling execution with Pixi as requested by command-line option `-without-pixi`"
config.pixi.enabled = false
}

// -- apply the pixi environment
if( cmdRun.withPixi ) {
if( cmdRun.withPixi != '-' )
config.process.pixi = cmdRun.withPixi
config.pixi.enabled = true
}

// -- sets the resume option
if( cmdRun.resume )
config.resume = cmdRun.resume
Expand Down Expand Up @@ -872,7 +885,7 @@ class ConfigBuilder {
final value = entry.value
final previous = getConfigVal0(config, key)
keys << entry.key

if( previous==null ) {
config[key] = value
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -335,6 +335,7 @@ class BashWrapperBuilder {
binding.before_script = getBeforeScriptSnippet()
binding.conda_activate = getCondaActivateSnippet()
binding.spack_activate = getSpackActivateSnippet()
binding.pixi_activate = getPixiActivateSnippet()

/*
* add the task environment
Expand Down Expand Up @@ -388,7 +389,7 @@ class BashWrapperBuilder {
binding.fix_ownership = fixOwnership() ? "[ \${NXF_OWNER:=''} ] && (shopt -s extglob; GLOBIGNORE='..'; chown -fR --from root \$NXF_OWNER ${workDir}/{*,.*}) || true" : null

binding.trace_script = isTraceRequired() ? getTraceScript(binding) : null

return binding
}

Expand Down Expand Up @@ -554,6 +555,29 @@ class BashWrapperBuilder {
return result
}

private String getPixiActivateSnippet() {
if( !pixiEnv )
return null
def result = "# pixi environment\n"

// Check if there's a .pixi file that points to the project directory
final pixiFile = pixiEnv.resolve('.pixi')
if( pixiFile.exists() ) {
// Read the project directory path
final projectDir = pixiFile.text.trim()
result += "cd ${Escape.path(projectDir as String)} && "
result += "eval \"\$(pixi shell-hook --shell bash)\" && "
result += "cd \"\$OLDPWD\"\n"
}
else {
// Direct activation from environment directory
result += "cd ${Escape.path(pixiEnv)} && "
result += "eval \"\$(pixi shell-hook --shell bash)\" && "
result += "cd \"\$OLDPWD\"\n"
}
return result
Comment on lines +565 to +578
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Where is OLDPWD defined? Doesn't this change to the projectDir? Should we overwrite projectDir with the pixiFile contents?

While it's uncessary duplication, I prefer copying the pixi lock file into the working dir and using it at source, so we have a record. This has the additional advantage of making the code more simple and supporting a .pixi or pixi.lock file (psuedocode):

Suggested change
if( pixiFile.exists() ) {
// Read the project directory path
final projectDir = pixiFile.text.trim()
result += "cd ${Escape.path(projectDir as String)} && "
result += "eval \"\$(pixi shell-hook --shell bash)\" && "
result += "cd \"\$OLDPWD\"\n"
}
else {
// Direct activation from environment directory
result += "cd ${Escape.path(pixiEnv)} && "
result += "eval \"\$(pixi shell-hook --shell bash)\" && "
result += "cd \"\$OLDPWD\"\n"
}
return result
// assuming you have copied the pixi file to working dir
if( pixiFile.exists() ) {
result += "[ -d .pixi ] && [ -f pixi.lock ] || pixi install"
result += "eval \"\$(pixi shell-hook --shell bash)\""
}
return result

Downside - massive duplication of text files in every work dir.

}

protected String getTraceCommand(String interpreter) {
String result = "${interpreter} ${fileStr(scriptFile)}"
if( input != null )
Expand Down Expand Up @@ -622,7 +646,7 @@ class BashWrapperBuilder {
private String copyFileToWorkDir(String fileName) {
copyFile(fileName, workDir.resolve(fileName))
}


String getCleanupCmd(String scratch) {
String result = ''
Expand Down
Loading