From 864eb29fc7a6682f8ddd332e4d524137cb347639 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 01:25:56 +0200 Subject: [PATCH 1/7] add julia skill --- scripts/julia-env-info | 94 +++++++++++++++++++++++++++++ skills/testing/julia-tests/SKILL.md | 72 ++++++++++++++++++++++ 2 files changed, 166 insertions(+) create mode 100755 scripts/julia-env-info create mode 100644 skills/testing/julia-tests/SKILL.md diff --git a/scripts/julia-env-info b/scripts/julia-env-info new file mode 100755 index 0000000..d0aaf25 --- /dev/null +++ b/scripts/julia-env-info @@ -0,0 +1,94 @@ +#!/usr/bin/env bash +set -euo pipefail +LC_ALL=C + +have() { command -v "$1" >/dev/null 2>&1; } + +# -------- parse julia_version from Manifest.toml (BSD/GNU portable) -------- +read_manifest_version() { + awk -F= ' + /^[[:space:]]*julia_version[[:space:]]*=/ { + v=$2 + gsub(/^[[:space:]]+|[[:space:]]+$/, "", v) + gsub(/^"|"$/, "", v) + print v + exit + } + ' "$1" 2>/dev/null || true +} + +# -------- parse juliaup api getconfig1 (no jq needed) -------- +read_juliaup_default() { + # Prints: "\t\t" OR nothing if unavailable + local cfg dc name ver file + cfg="$(juliaup api getconfig1 2>/dev/null || true)" || true + [ -z "${cfg:-}" ] && return 0 + # Extract the DefaultChannel object + dc="$(printf '%s' "$cfg" | awk 'match($0,/"DefaultChannel"[[:space:]]*:[[:space:]]*\{[^}]*\}/){print substr($0,RSTART,RLENGTH)}')" + [ -z "${dc:-}" ] && return 0 + # Pull fields (quoted strings) + name="$(printf '%s\n' "$dc" | grep -oE '"Name"[[:space:]]*:[[:space:]]*"[^"]*"' | sed -E 's/.*"Name"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/' || true)" + ver="$( printf '%s\n' "$dc" | grep -oE '"Version"[[:space:]]*:[[:space:]]*"[^"]*"' | sed -E 's/.*"Version"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/' || true)" + file="$(printf '%s\n' "$dc" | grep -oE '"File"[[:space:]]*:[[:space:]]*"[^"]*"' | sed -E 's/.*"File"[[:space:]]*:[[:space:]]*"([^"]*)".*/\1/' || true)" + [ -n "$name" ] && printf '%s\t%s\t%s\n' "$name" "$ver" "$file" +} + +# ---------- ENV ---------- +ENV_TSV="$(mktemp)"; PROJ_TSV="$(mktemp)"; MAN_TSV="$(mktemp)" +trap 'rm -f "$ENV_TSV" "$PROJ_TSV" "$MAN_TSV"' EXIT + +{ + printf 'KEY\tVALUE\n' + jpath="$(command -v julia 2>/dev/null || true)" + julia_present="no"; julia_ver="unknown" + if [ -n "${jpath:-}" ]; then + julia_present="yes" + if "$jpath" -e 'print(VERSION)' >/dev/null 2>&1; then + julia_ver="$("$jpath" -e 'print(VERSION)' 2>/dev/null || printf 'unknown')" + fi + fi + printf 'julia_present\t%s\n' "$julia_present" + printf 'julia_path\t%s\n' "${jpath:-none}" + printf 'julia_version\t%s\n' "$julia_ver" + + # juliaup details via API + juliaup_present="no"; uses_juliaup="no"; juliaup_default_channel="unknown"; juliaup_default_version="unknown"; juliaup_default_file="unknown" + if have juliaup; then + juliaup_present="yes" + case "${jpath:-}" in *".juliaup/"*) uses_juliaup="yes" ;; esac + if read -r ch ver file < <(read_juliaup_default); then + juliaup_default_channel="$ch" + [ -n "${ver:-}" ] && juliaup_default_version="$ver" + [ -n "${file:-}" ] && juliaup_default_file="$file" + fi + fi + printf 'juliaup_present\t%s\n' "$juliaup_present" + printf 'uses_juliaup\t%s\n' "$uses_juliaup" + printf 'juliaup_default_channel\t%s\n' "$juliaup_default_channel" + printf 'juliaup_default_version\t%s\n' "$juliaup_default_version" + printf 'juliaup_default_file\t%s\n' "$juliaup_default_file" +} >"$ENV_TSV" + +# ---------- PROJECTS ---------- +{ + printf 'PATH\n' + find . -type f -name 'Project.toml' -print0 2>/dev/null \ + | while IFS= read -r -d '' p; do printf '%s\n' "$p"; done +} >"$PROJ_TSV" + +# ---------- MANIFESTS ---------- +{ + printf 'PATH\tjulia_version\n' + find . -type f -name 'Manifest.toml' -print0 2>/dev/null \ + | while IFS= read -r -d '' m; do + v="$(read_manifest_version "$m")" + printf '%s\t%s\n' "$m" "${v:-unknown}" + done +} >"$MAN_TSV" + +# ---------- PRINT (pretty if `column` exists) ---------- +print_table() { if have column; then column -t -s $'\t' "$1"; else cat "$1"; fi; } + +printf 'ENV\n'; print_table "$ENV_TSV"; printf '\n' +printf 'PROJECTS\n'; print_table "$PROJ_TSV"; printf '\n' +printf 'MANIFESTS\n'; print_table "$MAN_TSV"; printf '\n' diff --git a/skills/testing/julia-tests/SKILL.md b/skills/testing/julia-tests/SKILL.md new file mode 100644 index 0000000..4aa0bbc --- /dev/null +++ b/skills/testing/julia-tests/SKILL.md @@ -0,0 +1,72 @@ +--- +name: Environments and tests in Julia +description: How to correctly activate environments and run tests in Julia +when_to_use: when running tests or struggling with project environments in Julia +version: 1.0.0 +languages: julia +--- + +## Julia projects + +Julia code is organized into projects, each with a Project.toml (name and dependencies) and potentially a Manifest.toml (lockfile). An environment might contain more than one Julia project in subfolders. + +When getting started with a Julia task, run + +```sh +${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info +``` + +to locate all the Project.toml and Manifest.toml files: + +```sh +ENV +KEY VALUE +julia_present yes +julia_path ${HOME}/.juliaup/bin/julia +julia_version 1.11.7 +juliaup_present yes +uses_juliaup yes +juliaup_default_channel release +juliaup_default_version 1.11.7 +juliaup_default_file ${HOME}/.julia/juliaup/julia-1.11.7+0.aarch64.apple.darwin14/bin/julia + +PROJECTS +PATH +./subdir/Project.toml +./docs/Project.toml + +MANIFESTS +PATH julia_version +./Manifest.toml 1.11.7 +./docs/Manifest.toml 1.11.7 +./subdir/Manifest.toml 1.11.7 +./subdir/test/Manifest.toml 1.11.7 +``` + +Then when starting Julia, pass the `--project` flag to specify the directory in which one of them is in. For example, + +```sh +julia --project=subdir -e 'using Pkg; Pkg.test()' +``` + +Basically every `julia` process you spawn should have a specified `--project` flag pointing to a directory with a project file in it. You can use `.` to refer to the current directory, or `..` to refer to the parent directory. If you want to create a new Julia project, again pass `--project` pointing to the directory you want to create the Julia project in, then Pkg to add dependencies, which will automatically generate the Project.toml file. + +Manifest.toml files are Julia-version-specific, and are automatically used when a package is activated to use the existing locked versions of dependencies. This can cause a conflict if the version of Julia you are using is different from the one that was used to resolve the Manifest.toml. That is why the `julia-env-info` script tells you the default version of Julia being used, as well as the one recorded in each Manifest.toml. + +In some cases, to resolve a version conflict it is OK to delete the Manifest.toml file. In other cases, it isn't. You should ask the user if you aren't sure before deleting a file. + +## Tests + +Most Julia projects use the Test stdlib for tests. Tests are organized into nestable testsets (`@testset`), and individual tests (`@test`). + +Here is how to get the docstring for `@test`: + +```sh +julia -e 'using Test; println(@doc @test)' +``` + +To run tests, you **need to activate the right environment** in order to get the test dependencies. There are a few ways to do this: + +1. Activate the project itself, and use `Pkg.test`. For example, `julia --project=subdir -e 'using Pkg; Pkg.test()'`. Then Pkg will figure out the right test environment to use. This will run _all_ of the tests, which may or may not be what you want, but it works basically every time since this is the universal entrypoint to testing in Julia. +2. Activate the test environment using TestEnv.jl. TestEnv.jl is a developer tool which uses the same machinery as Pkg to determine the correct test environment. The advantage here is that it does not run the tests automatically, so you can pick and choose what to run in that environment. For example: `julia --project=subdir -e 'using TestEnv; TestEnv.activate(); include("test/mytests.jl")'`. The problem here is that you need TestEnv to be installed globally as a developer tool for this to work. The user may or may not have it installed. You can ask if they want to install it. To install it globally, run the command `julia -e 'using Pkg; Pkg.add("TestEnv")'`. Here we have _not_ passed `--project`, so we use the global environment. That should be done rarely since most of the time we want the local one. Julia uses _stacked_ environments, so packages installed in the global one can be used when you have a local one installed (but may not be used as undeclared dependencies of a package; instead, add the dependency to the local package's environment if you want to use it from within the package). +3. Some projects have a `test/Project.toml` file. In that case, you can activate the test environment itself: `julia --project=subdir/test -e '...'`. However, TestEnv and Pkg can figure this out, so options (1) and (2) still work in this case. From f8e3d9f04ef04f8662f85638c098e98b718fc5bf Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 01:40:04 +0200 Subject: [PATCH 2/7] wip --- skills/testing/julia-tests/SKILL.md | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/skills/testing/julia-tests/SKILL.md b/skills/testing/julia-tests/SKILL.md index 4aa0bbc..9c3ad57 100644 --- a/skills/testing/julia-tests/SKILL.md +++ b/skills/testing/julia-tests/SKILL.md @@ -1,7 +1,7 @@ --- name: Environments and tests in Julia description: How to correctly activate environments and run tests in Julia -when_to_use: when running tests or struggling with project environments in Julia +when_to_use: when running Julia tests or struggling with project environments in Julia version: 1.0.0 languages: julia --- @@ -46,14 +46,24 @@ PATH julia_version Then when starting Julia, pass the `--project` flag to specify the directory in which one of them is in. For example, ```sh -julia --project=subdir -e 'using Pkg; Pkg.test()' +julia --project=subdir -e 'using Pkg; Pkg.instantiate()' ``` +to instantiate an existing environment (i.e. download dependencies, but do not mutate the environment or lockfile). + Basically every `julia` process you spawn should have a specified `--project` flag pointing to a directory with a project file in it. You can use `.` to refer to the current directory, or `..` to refer to the parent directory. If you want to create a new Julia project, again pass `--project` pointing to the directory you want to create the Julia project in, then Pkg to add dependencies, which will automatically generate the Project.toml file. -Manifest.toml files are Julia-version-specific, and are automatically used when a package is activated to use the existing locked versions of dependencies. This can cause a conflict if the version of Julia you are using is different from the one that was used to resolve the Manifest.toml. That is why the `julia-env-info` script tells you the default version of Julia being used, as well as the one recorded in each Manifest.toml. +Manifest.toml files are Julia-version-specific, and are automatically used when a package is activated to use the existing locked versions of dependencies. This can cause a conflict if the version of Julia you are using is different from the one that was used to resolve the Manifest.toml. That is why the `julia-env-info` script tells you the default version of Julia being used, as well as the one recorded in each Manifest.toml. + +Another common situation is that a Project.toml file will be checked-in to git, but not the Manifest.toml. For libraries, this is common practice, as we cannot specify the versions of all transitive dependencies for our users, as we are one of many libraries. In this case, the local Manifest.toml may get out of date with respect to a Project.toml that incorporates remote changes. When this happens, one gets the warning message: + +``` +Warning: The project dependencies or compat requirements have changed since the manifest was last resolved. +``` + +In this case, run `julia --project=my_project -e 'using Pkg; Pkg.resolve()'` to re-resolve the manifest using the latest Project.toml. -In some cases, to resolve a version conflict it is OK to delete the Manifest.toml file. In other cases, it isn't. You should ask the user if you aren't sure before deleting a file. +Additionally, in some cases, it is OK to delete the Manifest.toml file to start fresh. In other cases, it isn't. You should ask the user if you aren't sure before deleting a file. ## Tests From 81fa45a08fe699eacfc44d2e443c6161a1f4beb9 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:40:40 +0200 Subject: [PATCH 3/7] Improve julia-tests skill following TDD methodology MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applied writing-skills TDD process (RED-GREEN-REFACTOR): RED Phase - Baseline testing WITHOUT skill: - Created 8 test scenarios (retrieval, application, gap testing) - Documented failures: TestEnv.jl unknown, test/Project.toml missing, no Quick Reference, julia-env-info script unknown GREEN Phase - Initial improvements: - Added Overview with core principle - Added Quick Reference table with all common commands - Enriched when_to_use with error messages (LoadError, Manifest warnings) - Added TestEnv.jl section for running specific tests - Added Test-Only Dependencies with both patterns: * Pattern 1: test/Project.toml (modern) * Pattern 2: [extras] in main Project.toml - Added Common Issues section with cause→fix format - Added Common Mistakes table REFACTOR Phase - Concise rewrite: - Rewrote in concise style matching other skills - Reduced from 1043 to 626 words (40% reduction) - Added bold emphasis, clear numbered sections, tables - Maintained all essential information Verification - Re-ran all test scenarios: ✅ Scenario 1 (running tests): Found immediately, excellent scannability ✅ Scenario 4 (specific files): Correctly suggests TestEnv.jl ✅ Scenario 7 (test deps): Correctly identifies Pattern 1 vs Pattern 2 All scenarios PASS. Skill verified and ready for use. Version: 1.0.0 → 2.0.0 --- skills/testing/julia-tests/SKILL.md | 175 ++++++++++++++----- skills/testing/julia-tests/test-scenarios.md | 121 +++++++++++++ 2 files changed, 251 insertions(+), 45 deletions(-) create mode 100644 skills/testing/julia-tests/test-scenarios.md diff --git a/skills/testing/julia-tests/SKILL.md b/skills/testing/julia-tests/SKILL.md index 9c3ad57..789c16a 100644 --- a/skills/testing/julia-tests/SKILL.md +++ b/skills/testing/julia-tests/SKILL.md @@ -1,82 +1,167 @@ --- name: Environments and tests in Julia description: How to correctly activate environments and run tests in Julia -when_to_use: when running Julia tests or struggling with project environments in Julia -version: 1.0.0 +when_to_use: When running Julia tests, getting "LoadError" or package not found errors, seeing "Manifest out of date" warnings, struggling with --project flag, need to run specific test files, or want to add test-only dependencies. When working with Julia projects that have Project.toml/Manifest.toml files. +version: 2.0.0 languages: julia --- -## Julia projects +# Environments and Tests in Julia -Julia code is organized into projects, each with a Project.toml (name and dependencies) and potentially a Manifest.toml (lockfile). An environment might contain more than one Julia project in subfolders. +## Overview -When getting started with a Julia task, run +**Core principle:** Every Julia process needs `--project=DIR` to activate the correct environment. Tests need their own environment with test-specific dependencies. +**Julia projects:** Each has a `Project.toml` (dependencies) and optionally `Manifest.toml` (lockfile). + +## Quick Reference + +| Task | Command | +|------|---------| +| Find all projects | `${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info` | +| Run all tests | `julia --project=. -e 'using Pkg; Pkg.test()'` | +| Run specific test file | `julia --project=. -e 'using TestEnv; TestEnv.activate(); include("test/file.jl")'` | +| Fix "Manifest out of date" | `julia --project=. -e 'using Pkg; Pkg.resolve()'` | +| Instantiate environment | `julia --project=. -e 'using Pkg; Pkg.instantiate()'` | +| Add test-only dependency | `julia --project=test -e 'using Pkg; Pkg.add("Dep")'` OR `Pkg.add("Dep"; target=:extras)` | + +## Starting a Julia Task + +**1. Find all Julia projects in the repo:** ```sh ${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info ``` -to locate all the Project.toml and Manifest.toml files: +Shows: +- Julia version +- All Project.toml locations +- All Manifest.toml files with their Julia versions + +**2. Activate the right project:** +```sh +julia --project=subdir -e '...' # for subdir/Project.toml +julia --project=. -e '...' # for ./Project.toml +``` + +**Always use `--project` flag.** Without it, Julia uses the global environment. + +## Running Tests + +### Option 1: Run All Tests (Standard) + +```sh +julia --project=. -e 'using Pkg; Pkg.test()' +``` + +**Use this when:** +- Running full test suite +- CI/CD +- Not sure how tests are organized + +Pkg finds the test environment automatically. +### Option 2: Run Specific Test Files + +**Requirements:** +- TestEnv.jl installed globally: `julia -e 'using Pkg; Pkg.add("TestEnv")'` + +**Command:** ```sh -ENV -KEY VALUE -julia_present yes -julia_path ${HOME}/.juliaup/bin/julia -julia_version 1.11.7 -juliaup_present yes -uses_juliaup yes -juliaup_default_channel release -juliaup_default_version 1.11.7 -juliaup_default_file ${HOME}/.julia/juliaup/julia-1.11.7+0.aarch64.apple.darwin14/bin/julia - -PROJECTS -PATH -./subdir/Project.toml -./docs/Project.toml - -MANIFESTS -PATH julia_version -./Manifest.toml 1.11.7 -./docs/Manifest.toml 1.11.7 -./subdir/Manifest.toml 1.11.7 -./subdir/test/Manifest.toml 1.11.7 +julia --project=. -e 'using TestEnv; TestEnv.activate(); include("test/mytests.jl")' ``` -Then when starting Julia, pass the `--project` flag to specify the directory in which one of them is in. For example, +**Use this when:** +- Full suite takes too long +- Only changed specific code +- Developing test-first + +### Option 3: Direct Test Environment Activation +If `test/Project.toml` exists: ```sh -julia --project=subdir -e 'using Pkg; Pkg.instantiate()' +julia --project=test -e 'include("test/mytests.jl")' ``` -to instantiate an existing environment (i.e. download dependencies, but do not mutate the environment or lockfile). +Options 1 and 2 still work in this case. + +## Test-Only Dependencies + +Julia has two patterns: -Basically every `julia` process you spawn should have a specified `--project` flag pointing to a directory with a project file in it. You can use `.` to refer to the current directory, or `..` to refer to the parent directory. If you want to create a new Julia project, again pass `--project` pointing to the directory you want to create the Julia project in, then Pkg to add dependencies, which will automatically generate the Project.toml file. +### Pattern 1: test/Project.toml (Modern) -Manifest.toml files are Julia-version-specific, and are automatically used when a package is activated to use the existing locked versions of dependencies. This can cause a conflict if the version of Julia you are using is different from the one that was used to resolve the Manifest.toml. That is why the `julia-env-info` script tells you the default version of Julia being used, as well as the one recorded in each Manifest.toml. +**If `test/Project.toml` exists**, activate it: +```sh +julia --project=test -e 'using Pkg; Pkg.add("BenchmarkTools")' +``` + +### Pattern 2: [extras] in Main Project.toml -Another common situation is that a Project.toml file will be checked-in to git, but not the Manifest.toml. For libraries, this is common practice, as we cannot specify the versions of all transitive dependencies for our users, as we are one of many libraries. In this case, the local Manifest.toml may get out of date with respect to a Project.toml that incorporates remote changes. When this happens, one gets the warning message: +**If main Project.toml has `[extras]` section:** +**1. Add to [extras]:** +```sh +julia --project=. -e 'using Pkg; Pkg.add("BenchmarkTools"; target=:extras)' ``` -Warning: The project dependencies or compat requirements have changed since the manifest was last resolved. + +**2. Manually update [targets]:** +```toml +[extras] +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" + +[targets] +test = ["Test", "BenchmarkTools"] # Add BenchmarkTools here ``` -In this case, run `julia --project=my_project -e 'using Pkg; Pkg.resolve()'` to re-resolve the manifest using the latest Project.toml. +**Which to use:** Check what exists. Don't mix both. + +## Common Issues + +### "Manifest out of date" Warning -Additionally, in some cases, it is OK to delete the Manifest.toml file to start fresh. In other cases, it isn't. You should ask the user if you aren't sure before deleting a file. +**Cause:** Project.toml changed (e.g., after `git pull`) -## Tests +**Fix:** +```sh +julia --project=. -e 'using Pkg; Pkg.resolve()' +``` -Most Julia projects use the Test stdlib for tests. Tests are organized into nestable testsets (`@testset`), and individual tests (`@test`). +### Version Mismatches -Here is how to get the docstring for `@test`: +**Cause:** Manifest.toml created with different Julia version +**Check versions:** ```sh -julia -e 'using Test; println(@doc @test)' +${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info ``` -To run tests, you **need to activate the right environment** in order to get the test dependencies. There are a few ways to do this: +Shows your Julia version and each Manifest's Julia version. + +**Fix:** Re-resolve manifest or delete it (ask user first if it's checked in). + +### Tests Fail with LoadError + +**Cause:** Wrong environment activated or dependencies not installed + +**Fix:** +1. Check `--project` flag points to correct directory +2. Run `Pkg.instantiate()` to install dependencies +3. For tests, use `Pkg.test()` not direct file execution + +## Common Mistakes + +| Mistake | Fix | +|---------|-----| +| `julia test/runtests.jl` | `julia --project=. test/runtests.jl` | +| Deleting Manifest.toml carelessly | Ask user if it's checked in (may pin versions) | +| Missing `--project` flag | Add `--project=.` or `--project=DIR` | +| Forgetting to resolve after pull | Run `Pkg.resolve()` | + +## Environment Types + +**Global:** Used without `--project`. Install dev tools here (TestEnv.jl). + +**Local:** Activated with `--project`. Package dependencies. -1. Activate the project itself, and use `Pkg.test`. For example, `julia --project=subdir -e 'using Pkg; Pkg.test()'`. Then Pkg will figure out the right test environment to use. This will run _all_ of the tests, which may or may not be what you want, but it works basically every time since this is the universal entrypoint to testing in Julia. -2. Activate the test environment using TestEnv.jl. TestEnv.jl is a developer tool which uses the same machinery as Pkg to determine the correct test environment. The advantage here is that it does not run the tests automatically, so you can pick and choose what to run in that environment. For example: `julia --project=subdir -e 'using TestEnv; TestEnv.activate(); include("test/mytests.jl")'`. The problem here is that you need TestEnv to be installed globally as a developer tool for this to work. The user may or may not have it installed. You can ask if they want to install it. To install it globally, run the command `julia -e 'using Pkg; Pkg.add("TestEnv")'`. Here we have _not_ passed `--project`, so we use the global environment. That should be done rarely since most of the time we want the local one. Julia uses _stacked_ environments, so packages installed in the global one can be used when you have a local one installed (but may not be used as undeclared dependencies of a package; instead, add the dependency to the local package's environment if you want to use it from within the package). -3. Some projects have a `test/Project.toml` file. In that case, you can activate the test environment itself: `julia --project=subdir/test -e '...'`. However, TestEnv and Pkg can figure this out, so options (1) and (2) still work in this case. +**Stacked:** Local environment can use global packages, but packages can't declare undeclared global deps. diff --git a/skills/testing/julia-tests/test-scenarios.md b/skills/testing/julia-tests/test-scenarios.md new file mode 100644 index 0000000..84eba84 --- /dev/null +++ b/skills/testing/julia-tests/test-scenarios.md @@ -0,0 +1,121 @@ +# Test Scenarios for julia-tests Skill + +## Retrieval Scenarios + +### Scenario 1: First time running Julia tests +**Agent prompt:** +``` +I need to run tests for this Julia project. I see there's a test/ directory with test files. How do I run them? + +The project has: +- Project.toml in the root +- test/runtests.jl +- test/auth_tests.jl +``` + +**Success criteria:** +- Finds information about --project flag +- Discovers Pkg.test() as the standard approach +- Mentions this will run ALL tests + +### Scenario 2: Manifest out of date warning +**Agent prompt:** +``` +I pulled the latest changes and now when I start Julia I get: + +Warning: The project dependencies or compat requirements have changed since the manifest was last resolved. + +What should I do? +``` + +**Success criteria:** +- Finds the Pkg.resolve() solution +- Understands this resolves manifest from Project.toml +- Knows to use --project flag + +### Scenario 3: Multiple Julia projects in monorepo +**Agent prompt:** +``` +This repository seems to have Julia code in multiple directories: +- server/ +- client/ +- docs/ + +How do I figure out which ones are Julia projects and run their tests? +``` + +**Success criteria:** +- Finds julia-env-info script +- Understands it locates all Project.toml files +- Knows each needs its own --project activation + +## Application Scenarios + +### Scenario 4: Run specific test file +**Agent prompt:** +``` +The full test suite takes 10 minutes. I only changed authentication code. +Can you run just test/auth_tests.jl to verify my changes? + +Project structure: +mypackage/ + src/ + test/ + runtests.jl + auth_tests.jl + Project.toml +``` + +**Success criteria:** +- Uses TestEnv.jl OR activates test environment correctly +- Uses include("test/auth_tests.jl") to run specific file +- Explains trade-off (TestEnv needs to be installed) + +### Scenario 5: Clean test environment setup +**Agent prompt:** +``` +Tests are failing with weird dependency errors. Can you set up a fresh test environment from scratch? +``` + +**Success criteria:** +- Uses Pkg.test() approach (most reliable) +- Alternatively suggests checking if Manifest.toml can be deleted and re-resolved +- Asks user before deleting Manifest + +## Gap Testing + +### Scenario 6: Julia version mismatch +**Agent prompt:** +``` +I just upgraded from Julia 1.10 to Julia 1.11. Now when I try to run tests I get errors about package versions. What's wrong? +``` + +**Success criteria:** +- Explains Manifest.toml is version-specific +- Mentions julia-env-info shows both system version and manifest versions +- Suggests re-resolving manifest + +### Scenario 7: Adding test-only dependency +**Agent prompt:** +``` +I want to use BenchmarkTools.jl in my tests but not as a regular dependency. How do I add it as a test-only dependency? + +My project has a test/Project.toml file. +``` + +**Success criteria:** +- Finds the Test-Only Dependencies section +- Identifies Pattern 1 (test/Project.toml) is correct for this project +- Provides command: julia --project=test -e 'using Pkg; Pkg.add("BenchmarkTools")' +- Does NOT suggest [extras] approach for this project + +### Scenario 8: Global vs local package installation +**Agent prompt:** +``` +I want to install TestEnv.jl for development. Should I add it to my project or install it globally? +``` + +**Success criteria:** +- Explains stacked environments (covered briefly) +- Shows global install without --project flag +- Explains when to use global vs local From cf5330f2a1653159b2036ea826a27e043f8d1389 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:49:53 +0200 Subject: [PATCH 4/7] tweaks --- skills/testing/julia-tests/SKILL.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/skills/testing/julia-tests/SKILL.md b/skills/testing/julia-tests/SKILL.md index 789c16a..1919116 100644 --- a/skills/testing/julia-tests/SKILL.md +++ b/skills/testing/julia-tests/SKILL.md @@ -88,7 +88,7 @@ Options 1 and 2 still work in this case. Julia has two patterns: -### Pattern 1: test/Project.toml (Modern) +### Pattern 1: test/Project.toml **If `test/Project.toml` exists**, activate it: ```sh @@ -138,7 +138,7 @@ ${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info Shows your Julia version and each Manifest's Julia version. -**Fix:** Re-resolve manifest or delete it (ask user first if it's checked in). +**Fix:** Re-resolve manifest or delete it (ask user before deleting). ### Tests Fail with LoadError @@ -154,7 +154,7 @@ Shows your Julia version and each Manifest's Julia version. | Mistake | Fix | |---------|-----| | `julia test/runtests.jl` | `julia --project=. test/runtests.jl` | -| Deleting Manifest.toml carelessly | Ask user if it's checked in (may pin versions) | +| Deleting Manifest.toml carelessly | Ask user before deleting | | Missing `--project` flag | Add `--project=.` or `--project=DIR` | | Forgetting to resolve after pull | Run `Pkg.resolve()` | From bd09ef0209bc20974b3f0c3270f9a8d3947ca00c Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 15:50:52 +0200 Subject: [PATCH 5/7] Update scripts/julia-env-info Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- scripts/julia-env-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/julia-env-info b/scripts/julia-env-info index d0aaf25..2dc6ba5 100755 --- a/scripts/julia-env-info +++ b/scripts/julia-env-info @@ -1,6 +1,6 @@ #!/usr/bin/env bash set -euo pipefail -LC_ALL=C +export LC_ALL=C have() { command -v "$1" >/dev/null 2>&1; } From f799fb098c3bb177414592a04112f23612285ab1 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 16:38:32 +0200 Subject: [PATCH 6/7] get docs, test_args --- skills/testing/julia-tests/SKILL.md | 6 ++- skills/testing/julia-tests/test-scenarios.md | 45 ++++++++++++++++++++ 2 files changed, 49 insertions(+), 2 deletions(-) diff --git a/skills/testing/julia-tests/SKILL.md b/skills/testing/julia-tests/SKILL.md index 1919116..dbd0309 100644 --- a/skills/testing/julia-tests/SKILL.md +++ b/skills/testing/julia-tests/SKILL.md @@ -1,8 +1,8 @@ --- name: Environments and tests in Julia description: How to correctly activate environments and run tests in Julia -when_to_use: When running Julia tests, getting "LoadError" or package not found errors, seeing "Manifest out of date" warnings, struggling with --project flag, need to run specific test files, or want to add test-only dependencies. When working with Julia projects that have Project.toml/Manifest.toml files. -version: 2.0.0 +when_to_use: When running Julia tests, getting "LoadError" or package not found errors, seeing "Manifest out of date" warnings, struggling with --project flag, need to run specific test files, or want to add test-only dependencies. When working with Julia projects that have Project.toml/Manifest.toml files. When you need to find Julia documentation for functions or understand what test_args does. +version: 2.1.0 languages: julia --- @@ -19,6 +19,7 @@ languages: julia | Task | Command | |------|---------| | Find all projects | `${SUPERPOWERS_SKILLS_ROOT}/scripts/julia-env-info` | +| Get function documentation | `julia -e 'using Pkg; println(@doc Pkg.test)'` | | Run all tests | `julia --project=. -e 'using Pkg; Pkg.test()'` | | Run specific test file | `julia --project=. -e 'using TestEnv; TestEnv.activate(); include("test/file.jl")'` | | Fix "Manifest out of date" | `julia --project=. -e 'using Pkg; Pkg.resolve()'` | @@ -154,6 +155,7 @@ Shows your Julia version and each Manifest's Julia version. | Mistake | Fix | |---------|-----| | `julia test/runtests.jl` | `julia --project=. test/runtests.jl` | +| `Pkg.test(test_args=["file.jl"])` | `test_args` are command-line arguments (ARGS), NOT file selectors. Use TestEnv to run specific files. | | Deleting Manifest.toml carelessly | Ask user before deleting | | Missing `--project` flag | Add `--project=.` or `--project=DIR` | | Forgetting to resolve after pull | Run `Pkg.resolve()` | diff --git a/skills/testing/julia-tests/test-scenarios.md b/skills/testing/julia-tests/test-scenarios.md index 84eba84..25fd197 100644 --- a/skills/testing/julia-tests/test-scenarios.md +++ b/skills/testing/julia-tests/test-scenarios.md @@ -119,3 +119,48 @@ I want to install TestEnv.jl for development. Should I add it to my project or i - Explains stacked environments (covered briefly) - Shows global install without --project flag - Explains when to use global vs local + +### Scenario 9: Finding Julia documentation +**Agent prompt:** +``` +I need to check the documentation for Pkg.test to understand how the test_args parameter works. How do I get the docs for this function? + +Working directory: /Users/eph/.config/superpowers/skills +``` + +**Success criteria:** +- Uses julia -e 'using Pkg; println(@doc Pkg.test)' command +- Finds the Quick Reference entry for getting documentation +- Explains that @doc shows version-specific docs from the installed Julia version + +**Pressure points:** +- Time pressure: User needs quick answer +- Unfamiliarity: Agent may not know Julia doc syntax +- Alternative temptation: Search web instead of checking actual installed version + +### Scenario 10: test_args misconception +**Agent prompt:** +``` +I only changed the parsing logic. Can you just run the test/parser_tests.jl file instead of the whole suite to save time? + +Context: +- Project has standard structure with Project.toml in root +- test/runtests.jl exists +- test/parser_tests.jl exists +- TestEnv.jl is installed globally +``` + +**Success criteria:** +- Uses TestEnv approach: julia --project=. -e 'using TestEnv; TestEnv.activate(); include("test/parser_tests.jl")' +- Does NOT attempt: Pkg.test(test_args=["parser_tests.jl"]) +- If considering test_args, explicitly rejects it with correct reasoning +- Explains test_args are command-line arguments (ARGS), not file selectors + +**Common failure mode:** +Agent incorrectly tries: Pkg.test(test_args=["parser_tests.jl"]) +This fails because test_args passes strings to ARGS in the test process, not file paths to run. + +**Pressure points:** +- Time pressure: User wants faster feedback +- Seeming logic: Parameter name "test_args" sounds like it could specify files +- Documentation complexity: Need to understand what test_args actually does From 64a5d17030c5a19f68a7147fba774062e1ade4e8 Mon Sep 17 00:00:00 2001 From: Eric Hanson <5846501+ericphanson@users.noreply.github.com> Date: Mon, 13 Oct 2025 16:53:38 +0200 Subject: [PATCH 7/7] Update scripts/julia-env-info Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- scripts/julia-env-info | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/julia-env-info b/scripts/julia-env-info index 2dc6ba5..269d643 100755 --- a/scripts/julia-env-info +++ b/scripts/julia-env-info @@ -56,7 +56,7 @@ trap 'rm -f "$ENV_TSV" "$PROJ_TSV" "$MAN_TSV"' EXIT if have juliaup; then juliaup_present="yes" case "${jpath:-}" in *".juliaup/"*) uses_juliaup="yes" ;; esac - if read -r ch ver file < <(read_juliaup_default); then + if IFS=$'\t' read -r ch ver file < <(read_juliaup_default); then juliaup_default_channel="$ch" [ -n "${ver:-}" ] && juliaup_default_version="$ver" [ -n "${file:-}" ] && juliaup_default_file="$file"