Merge pull request #6 from jeremymanning/main

Repository reorganization and training improvements

Author: jeremymanning

.github/workflows/tests.yml

@@ -63,11 +63,11 @@ jobs:
 
     - name: Test figure generation (single)
       run: |
-        python generate_figures.py --figure 1a --data tests/data/test_model_results.pkl --output tests/output_single
+        python code/generate_figures.py --figure 1a --data tests/data/test_model_results.pkl --output tests/output_single
 
     - name: Test figure generation (all)
       run: |
-        python generate_figures.py --data tests/data/test_model_results.pkl --output tests/output_all
+        python code/generate_figures.py --data tests/data/test_model_results.pkl --output tests/output_all
       timeout-minutes: 5
 
     - name: Upload test artifacts

README.md

@@ -21,15 +21,16 @@ llm-stylometry/
 │   ├── utils/          # Helper utilities
 │   ├── visualization/  # Plotting and visualization
 │   └── cli_utils.py    # CLI helper functions
-├── code/                # Original analysis scripts
-│   ├── main.py         # Model training script
+├── code/                # Training and CLI scripts
+│   ├── generate_figures.py       # Main CLI entry point
+│   ├── consolidate_model_results.py # Result consolidation
+│   ├── main.py         # Model training orchestration
 │   ├── clean.py        # Data preprocessing
-│   └── ...             # Various analysis scripts
+│   └── ...             # Supporting training modules
 ├── data/                # Datasets and results
 │   ├── raw/            # Original texts from Project Gutenberg
 │   ├── cleaned/        # Preprocessed texts by author
-│   ├── model_results.pkl # Consolidated model training results
-│   └── model_results.csv # Model results in CSV format
+│   └── model_results.pkl # Consolidated model training results
 ├── models/              # Trained models (80 total)
 │   └── {author}_tokenizer=gpt2_seed={0-9}/
 ├── paper/               # LaTeX paper and figures
@@ -40,7 +41,6 @@ llm-stylometry/
 │   ├── data/           # Test data and fixtures
 │   ├── test_*.py       # Test modules
 │   └── check_outputs.py # Output validation script
-├── generate_figures.py  # Main CLI entry point
 ├── run_llm_stylometry.sh # Shell wrapper for easy setup
 ├── LICENSE             # MIT License
 ├── README.md           # This file
@@ -168,16 +168,17 @@ fig = generate_all_losses_figure(
 **Note**: Training requires a CUDA-enabled GPU and takes significant time (~80 models total).
 
 ```bash
-# Using the CLI (recommended)
+# Using the CLI (recommended - handles all steps automatically)
 ./run_llm_stylometry.sh --train
-
-# Or manually
-conda activate llm-stylometry
-python code/clean.py              # Clean data
-python code/main.py               # Train models
-python consolidate_model_results.py  # Consolidate results
 ```
 
+This command will:
+1. Clean and prepare the data if needed
+2. Train all 80 models (8 authors × 10 seeds)
+3. Consolidate results into `data/model_results.pkl`
+
+The training pipeline automatically handles data preparation, model training across available GPUs, and result consolidation. Individual model checkpoints and loss logs are saved in the `models/` directory.
+
 ### Model Configuration
 
 Each model uses: