R2dt rescan

config/r2dt.config

@@ -4,6 +4,7 @@ params {
     sequence_chunks = 4000
     data_chunk_size = 1024 * 1000 * 1000
     sequence_chunk_size = 1000
+    sequence_count = 2000000
     tablename = 'traveler_sequences_to_analyze'
     publish = "$baseDir/r2dt"
     container = 'rnacentral/r2dt:latest'

files/r2dt/attempted.ctl

@@ -1,11 +1,13 @@
 LOAD CSV
 FROM ALL FILENAMES MATCHING ~<r2dt-attempted.*csv$>
 HAVING FIELDS (
-  urs
+  urs,
+  r2dt_version
 )
 INTO {{PGDATABASE}}?load_traveler_attempted
 TARGET COLUMNS (
-  urs
+  urs,
+  r2dt_version
 )
 
 WITH
@@ -19,23 +21,27 @@ DROP TABLE IF EXISTS load_traveler_attempted;
 $$,
 $$
 CREATE TABLE load_traveler_attempted (
-  urs text primary key
+  urs text primary key,
+  r2dt_version text
 );
 $$
 
 AFTER LOAD DO
 $$
 INSERT INTO pipeline_tracking_traveler (
   urs,
-  last_run
+  last_run,
+  r2dt_version
 ) (
 SELECT
   load.urs,
-  NOW()
+  NOW(),
+  load.r2dt_version
 FROM load_traveler_attempted load
 ) ON CONFLICT (urs) DO UPDATE
-SET 
-  last_run = EXCLUDED.last_run
+SET
+  last_run = EXCLUDED.last_run,
+  r2dt_version = EXCLUDED.r2dt_version
 ;
 $$
 ;

files/r2dt/find-sequences.sql

@@ -5,7 +5,8 @@ SELECT
     'sequence', COALESCE(rna.seq_short, rna.seq_long)
   )
 FROM rna
-WHERE 
+WHERE
   not exists(select 1 from pipeline_tracking_traveler track where track.urs = rna.upi)
   AND rna.len < :max_len
+  LIMIT :sequence_count
 ) TO STDOUT;

files/r2dt/load-models.ctl

@@ -1,21 +1,21 @@
 LOAD CSV
-FROM ALL FILENAMES MATCHING ~<models.*.csv$>
+FROM ALL FILENAMES MATCHING ~<data.*.csv$>
 HAVING FIELDS (
     model_name,
     taxid,
+    cellular_location,
     rna_type,
-    so_term,
-    cell_location,
+    so_term_id,
     model_source,
     model_length,
     model_basepair_count
 ) INTO {{PGDATABASE}}?load_secondary_layout_models
 TARGET COLUMNS (
     model_name,
     taxid,
+    cellular_location,
     rna_type,
-    so_term,
-    cell_location,
+    so_term_id,
     model_source,
     model_length,
     model_basepair_count
@@ -33,9 +33,9 @@ $$
 create table load_secondary_layout_models (
     model_name text NOT NULL,
     taxid int NOT NULL,
+    cellular_location text,
     rna_type text NOT NULL,
-    so_term text NOT NULL,
-    cell_location text,
+    so_term_id text NOT NULL,
     model_source text not null,
     model_length int,
     model_basepair_count int
@@ -47,24 +47,33 @@ $$
 INSERT INTO rnc_secondary_structure_layout_models (
     model_name,
     taxid,
+    cellular_location,
     rna_type,
     so_term_id,
-    cellular_location,
     model_source,
     model_length,
     model_basepair_count
 ) (
 SELECT
     model_name,
     taxid,
+    cellular_location,
     rna_type,
-    so_term,
-    cell_location,
+    so_term_id,
     model_source,
     model_length,
     model_basepair_count
 FROM load_secondary_layout_models load
-) ON CONFLICT (model_name) DO NOTHING
+) ON CONFLICT (model_name) DO UPDATE
+SET
+    taxid = EXCLUDED.taxid,
+    cellular_location = EXCLUDED.cellular_location,
+    rna_type = EXCLUDED.rna_type,
+    so_term_id = EXCLUDED.so_term_id,
+    model_source = EXCLUDED.model_source,
+    model_length = EXCLUDED.model_length,
+    model_basepair_count = EXCLUDED.model_basepair_count
+
 ;
 $$,
 $$

r2dt-scan.nf

@@ -0,0 +1,130 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl=2
+
+include { r2dt } from './workflows/r2dt'
+
+
+process parse_gtrnadb_model {
+
+  input:
+    path(model_path)
+  output:
+    path("model_data.csv")
+
+  script:
+    """
+    rnac r2dt model-info gtrnadb $model_path model_data.csv
+    """
+}
+
+process parse_ribovision_models {
+
+  input:
+    val(ribovision_metadata_url)
+
+
+
+  output:
+    path("model_data.csv")
+
+  script:
+  """
+  wget $ribovision_metadata_url
+
+  rnac r2dt model-info ribovision metadata.tsv model_data.csv
+  """
+
+}
+
+process parse_rnasep_models {
+
+  input:
+    val(rnasep_metadata_url)
+
+  output:
+    path("model_data.csv")
+
+  script:
+  """
+  wget $rnasep_metadata_url
+  sed -i 's/\\tNRC-1\\t/\\t/g' metadata.tsv
+  rnac r2dt model-info rnase-p metadata.tsv model_data.csv
+  """
+
+}
+
+process parse_rfam_models {
+
+  input:
+    path(all_models)
+  output:
+    path("model_data.csv")
+
+  script:
+  """
+  rnac r2dt model-info rfam $all_models $PGDATABASE model_data.csv
+  """
+}
+
+
+process parse_crw_models {
+
+  input:
+    tuple path(all_models), val(metadata)
+  output:
+    path("model_data.csv")
+
+  script:
+  """
+  wget $metadata -O metadata.tsv
+  sed -i 's/taxid  rna_type/taxid\trna_type/g' metadata.tsv
+  rnac r2dt model-info crw $all_models metadata.tsv model_data.csv
+  """
+}
+
+process load_models {
+
+  input:
+    path(all_data)
+    path(ctl)
+
+  output:
+    val('models loaded')
+
+  script:
+  """
+  split-and-load $ctl $all_data ${params.import_data.chunk_size}
+  """
+}
+
+
+
+
+
+workflow {
+  rfam_models = Channel.of("$baseDir/singularity/bind/r2dt/data/cms/rfam/all.cm")
+  crw_models = Channel.of("$baseDir/singularity/bind/r2dt/data/cms/crw/all.cm")
+  crw_metadata = Channel.of("https://raw.githubusercontent.com/RNAcentral/R2DT/v1.3/data/crw-metadata.tsv")
+  gtrnadb_models = Channel.fromPath("$baseDir/singularity/bind/r2dt/data/cms/gtrnadb/*.cm")
+  ribovision_lsu_metadata_url = Channel.of("https://raw.githubusercontent.com/RNAcentral/R2DT/v1.3/data/ribovision-lsu/metadata.tsv")
+  ribovision_ssu_metadata_url = Channel.of("https://raw.githubusercontent.com/RNAcentral/R2DT/v1.3/data/ribovision-ssu/metadata.tsv")
+
+  rnasep_metadata_url = Channel.of("https://raw.githubusercontent.com/RNAcentral/R2DT/v1.3/data/rnasep/metadata.tsv")
+
+  load_ctl = Channel.of("$baseDir/files/r2dt/load-models.ctl")
+
+  rfam_models | parse_rfam_models | set { rfam_data }
+  crw_models.combine(crw_metadata) | parse_crw_models | set { crw_data }
+  gtrnadb_models | parse_gtrnadb_model | collectFile() {csvfile -> [csvfile.name, csvfile.text]} | set { gtrnadb_data }
+  ribovision_lsu_metadata_url.mix(ribovision_ssu_metadata_url) | parse_ribovision_models | set {ribovision_data }
+  rnasep_metadata_url | parse_rnasep_models | set {rnasep_data}
+
+  rfam_data.mix(crw_data, gtrnadb_data, ribovision_data, rnasep_data) | collectFile() {csvfile -> [csvfile.name, csvfile.text]} | set { all_data }
+
+
+   load_models(all_data, load_ctl) | set { model_load }
+
+   model_load | r2dt | set { done }
+
+}

rnacentral_pipeline/cli/r2dt.py

@@ -138,12 +138,13 @@ def model_info():
 
 @model_info.command("crw")
 @click.argument("filename", type=click.File("r"))
+@click.argument("metadata_url", type=str)
 @click.argument("output", default="-", type=click.File("w"))
-def crw_model_info(filename, output):
+def crw_model_info(filename, metadata_url, output):
     """
     Parse the CRW metadata file and produce
     """
-    r2dt.write_crw(filename, output)
+    r2dt.write_crw(filename, metadata_url, output)
 
 
 @model_info.command("ribovision")
@@ -158,7 +159,7 @@ def ribovision_model_info(filename, output):
 
 
 @model_info.command("gtrnadb")
-@click.argument("filename", type=click.File("r"))
+@click.argument("filename", type=click.File())
 @click.argument("output", default="-", type=click.File("w"))
 def gtrnadb_model_info(filename, output):
     """
@@ -181,20 +182,23 @@ def rnase_p_model_info(filename, output):
 
 @model_info.command("rfam")
 @click.argument("filename", type=click.File("r"))
+@click.argument("db_url", type=str)
 @click.argument("output", default="-", type=click.File("w"))
-def rnase_p_model_info(filename, output):
+def rnase_p_model_info(filename, db_url, output):
     """
     Parse the metadata.tsv file from R2DT for Ribovision models to
     produce something we can put in our database.
     """
-    r2dt.write_rfam(filename, output)
+    r2dt.write_rfam(filename, db_url, output)
 
 
 @cli.command("create-attempted")
 @click.argument("filename", type=click.File("r"))
+@click.argument("version", type=click.File("r"))
 @click.argument("output", default="-", type=click.File("w"))
-def r2dt_create_attempted(filename, output):
-    attempted.r2dt(filename, output)
+def r2dt_create_attempted(filename, version, output):
+    version_string = version.read().strip()
+    attempted.r2dt(filename, version_string, output)
 
 
 @cli.command("publish")

rnacentral_pipeline/rnacentral/attempted.py

@@ -62,10 +62,14 @@ def parse_rfam_version(handle: ty.IO) -> str:
     raise ValueError(f"Could not find version in file {handle}")
 
 
-def write(data: ty.Iterable[ty.List[str]], output: ty.IO, require_attempt=True):
+def write(
+    data: ty.Iterable[ty.List[str]], output: ty.IO, require_attempt=True, version=None
+):
     writer = csv.writer(output)
     seen = False
     for row in data:
+        if version:
+            row.append(version)
         writer.writerow(row)
         seen = True
     if not seen:
@@ -88,6 +92,6 @@ def qa(handle: ty.IO, name: str, version_file: ty.IO, output: ty.IO):
     write(data, output)
 
 
-def r2dt(handle: ty.IO, output: ty.IO):
+def r2dt(handle: ty.IO, version: str, output: ty.IO):
     data = fasta_parser(handle)
-    write(data, output)
+    write(data, output, version=version)