try moving away from biobear due to rust-related installation issues on some platforms

Author: nickzoic

countess/plugins/fastq.py

@@ -1,8 +1,10 @@
+import itertools
 import logging
 from typing import Iterable, Optional
 
-import biobear
+import dnaio
 import duckdb
+import pyarrow
 
 from countess import VERSION
 from countess.core.parameters import BaseParam, BooleanParam, FloatParam, StringParam
@@ -32,10 +34,20 @@ def load_file(
     ) -> duckdb.DuckDBPyRelation:
         # Open the file, convert it to a RecordBatchReader and then
         # wrap that up as a DuckDBPyRelation so we can filter it.
-        reader = biobear.connect().read_fastq_file(filename)
-        rel = cursor.from_arrow(reader.to_arrow_record_batch_reader())
+        fastq_iter = dnaio.open(filename, open_threads=1)
+        record_batch_iter = (
+            pyarrow.RecordBatch.from_pylist([{'sequence': z.sequence, 'quality_scores': z.qualities} for z in y])
+            for y in itertools.batched(fastq_iter, 5000)
+        )
+        rel = cursor.from_arrow(
+            pyarrow.RecordBatchReader.from_batches(
+                pyarrow.schema({'sequence': 'str', 'quality_scores': 'str'}),
+                record_batch_iter
+            )
+        )
         if row_limit is not None:
-            rel = rel.limit(row_limit)
+            pass
+        #rel = rel.limit(row_limit)
 
         if self.min_avg_quality > 0:
             rel = rel.filter(
@@ -77,8 +89,17 @@ class LoadFastaPlugin(DuckdbLoadFileWithTheLotPlugin):
     def load_file(
         self, cursor: duckdb.DuckDBPyConnection, filename: str, file_param: BaseParam, row_limit: Optional[int] = None
     ) -> duckdb.DuckDBPyRelation:
-        reader = biobear.connect().read_fasta_file(filename)
-        rel = cursor.from_arrow(reader.to_arrow_record_batch_reader())
+        fasta_iter = dnaio.open(filename, open_threads=1)
+        record_batch_iter = (
+            pyarrow.RecordBatch.from_pylist([{'seq': z.sequence, 'qual': z.qualities} for z in y])
+            for y in itertools.batched(fasta_iter, 5000)
+        )
+        rel = cursor.from_arrow(
+            pyarrow.RecordBatchReader.from_batches(
+                pyarrow.schema({'seq': 'str', 'qual': 'str'}),
+                record_batch_iter
+            )
+        )
         if row_limit is not None:
             rel = rel.limit(row_limit)
         return rel

docs/installing-countess/index.md

@@ -97,9 +97,10 @@ we use [homebrew](https://brew.sh/) to install more recent versions.
   * Run `rustup-init.exe` to install rust and cargo.
 
   * If you haven't already got VS Code installed, select 1 to install it.
-    Once it is installed, rustup will ask you to reboot.
-    Once your machine has rebooted run `rustup-init.exe` again to finish
-    installation of rust and cargo.
+
+    * Ignore the prompt to log in to Visual Studio and just let rustup finish.
+    * Once it is installed, run `rustup-init.exe` and select 1 again.
+    * This time it should say "Rust is installed now. Great!" 
 
   * now open `cmd.exe` and run `pip install countess`.
 

pyproject.toml

@@ -18,7 +18,7 @@ classifiers = [
     'Topic :: Scientific/Engineering :: Bio-Informatics',
 ]
 dependencies = [
-    'biobear~=0.23.2',
+    'dnaio~=1.2.3',
     'duckdb>=1.3.1',
     'fqfa~=1.3.1',
     'more_itertools~=9.1.0',