Merge pull request #8 from antgonza/Tell-Seq

Tell-Seq

Author: charles-cowart

.github/workflows/qiita-plugin-ci.yml

@@ -2,7 +2,7 @@
 
 on:
   push:
-    branches: [ dev ]
+    branches: [dev]
   pull_request:
 
 jobs:
@@ -39,7 +39,7 @@ jobs:
         uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: 3.6
+          python-version: 3.9
 
       - name: Basic dependencies install
         env:
@@ -62,15 +62,15 @@ jobs:
 
           # Setting up main qiita conda environment
           conda config --add channels conda-forge
-          conda create -q --yes -n qiita python=3.6 pip libgfortran numpy nginx cython redis
+          conda create --quiet --yes -n qiita python=3.9 pip libgfortran numpy nginx cython redis
           conda activate qiita
           pip install sphinx sphinx-bootstrap-theme nose-timer codecov Click
 
       - name: Qiita install
         shell: bash -l {0}
         run: |
           conda activate qiita
-          pip install qiita-dev/ --no-binary redbiom
+          pip --quiet install qiita-dev/ --no-binary redbiom
           mkdir ~/.qiita_plugins
 
       - name: Install plugins
@@ -80,37 +80,34 @@ jobs:
           conda config --add channels bioconda
 
           # installing qtp-sequencing
-          conda create -q --yes -n qtp-sequencing python=3.6 pip pigz quast
+          conda create -q --yes -n qtp-sequencing python=3.9 pip pigz quast
           conda activate qtp-sequencing
           pip --quiet install https://github.com/qiita-spots/qtp-sequencing/archive/master.zip
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
+          export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
-          configure_qtp_sequencing --env-script "source /home/runner/.profile; conda activate qtp-sequencing" --server-cert $QIITA_SERVER_CERT
-
+          configure_qtp_sequencing --env-script "source /home/runner/.profile; conda activate qtp-sequencing" --server-cert $QIITA_ROOTCA_CERT
           conda deactivate
 
           # installing this plugin
-          conda create -q --yes -n qp-spades python=3.6 cmake clang zlib flash spades=3.15.2
+          conda create -q --yes -n qp-spades python=3.9 cmake clang zlib flash
           conda activate qp-spades
 
           # making sure the installed versions haven't changed
           flash_version=`flash --version`
-          spades_version=`spades.py --version`
           if [[ $flash_version != *"v1.2.11"* ]]; then echo "wrong flash version", $flash_version; exit 1; fi
-          if [[ $spades_version != *"v3.15.2"* ]]; then echo "wrong spades version", $spades_version; exit 1; fi
 
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
+          export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
           export ENVIRONMENT="source /home/runner/.profile; conda activate qp-spades"
           pip install -U pip
           pip install .
-          configure_qp_spades --env-script 'source /home/runner/.profile; conda activate qp-spades; export ENVIRONMENT="source /home/runner/.profile; conda activate qp-spades"' --server-cert $QIITA_SERVER_CERT
+          configure_qp_spades --env-script 'source /home/runner/.profile; conda activate qp-spades; export ENVIRONMENT="source /home/runner/.profile; conda activate qp-spades"' --ca-cert $QIITA_ROOTCA_CERT
 
       - name: Starting services
         shell: bash -l {0}
         run: |
           conda activate qiita
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
+          export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
           sed "s#/home/runner/work/qiita/qiita#${PWD}/qiita-dev/#g" `pwd`/qiita-dev/qiita_core/support_files/config_test.cfg > ${QIITA_CONFIG_FP}
 
@@ -143,11 +140,11 @@ jobs:
           COVER_PACKAGE: ${{ matrix.cover_package }}
         run: |
           conda activate qp-spades
-          export QIITA_SERVER_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
+          export QIITA_ROOTCA_CERT=`pwd`/qiita-dev/qiita_core/support_files/ci_rootca.crt
           export QIITA_CONFIG_FP=`pwd`/qiita-dev/qiita_core/support_files/config_test_local.cfg
           export ENVIRONMENT="source /home/runner/.profile; conda activate qp-spades"
 
-          nosetests $COVER_PACKAGE --with-doctest --with-coverage -vv --cover-package=$COVER_PACKAGE
+          nosetests --with-doctest --with-coverage -v --cover-package=qp_woltka
 
           # killing the qiita server to run the next commands
           QIITA_PID=`cat /tmp/supervisord.pid`
@@ -156,21 +153,21 @@ jobs:
 
       - uses: codecov/codecov-action@v1
         with:
-         token: ${{ secrets.CODECOV_TOKEN }}
-         file: codecov.yml
+          token: ${{ secrets.CODECOV_TOKEN }}
+          file: codecov.yml
 
   lint:
     runs-on: ubuntu-latest
     steps:
-    - name: flake8
-      uses: actions/setup-python@v2
-      with:
-        python-version: 3.9
-    - name: install dependencies
-      run: python -m pip install --upgrade pip
-    - name: Check out repository code
-      uses: actions/checkout@v2
-    - name: lint
-      run: |
-        pip install -q flake8
-        flake8 qp_spades setup.py scripts/*
+      - name: flake8
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - name: install dependencies
+        run: python -m pip install --upgrade pip
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: lint
+        run: |
+          pip install -q flake8
+          flake8 qp_spades setup.py scripts/*

qp_spades/__init__.py

@@ -10,13 +10,11 @@
 from .qp_spades import spades
 
 
-THREADS = "16"
-MEMORY = "200"
-KMERS = "21,33,55,77,99,127"
+THREADS = "12"
 
 plugin_details = {'name': 'qp-spades',
-                  'version': '3.15.2',
-                  'description': 'spades pipeline'}
+                  'version': '2025.02',
+                  'description': 'spades assembler pipeline'}
 
 
 # Initialize the plugin
@@ -25,30 +23,19 @@
 # Define the command
 req_params = {'input': ('artifact', ['per_sample_FASTQ'])}
 opt_params = {
-    'type': ['choice:["meta", "isolate"]', "meta"],
-    'merging': ['choice:["no merge", "flash 65%"]', "flash 65%"],
+    'type': ['choice:["meta"]', "meta"],
+    'merging': ['choice:["no merge", "flash 65%"]', "no merge"],
     'threads': ['integer', THREADS],
-    'memory': ['integer', MEMORY],
-    'k-mers': ['string', KMERS],
     }
 
 outputs = {'Preprocessed FASTA': 'FASTA_preprocessed'}
 default_params = {
-    'no merging + meta': {
-        'type': 'meta', 'merging': 'no merge', 'threads': THREADS,
-        'memory': MEMORY, 'k-mers': KMERS},
-    'no merging + isolate': {
-        'type': 'isolate', 'merging': 'no merge', 'threads': THREADS,
-        'memory': MEMORY, 'k-mers': KMERS},
-    'merge flash 65% + meta': {
-        'type': 'meta', 'merging': 'flash 65%', 'threads': THREADS,
-        'memory': MEMORY, 'k-mers': KMERS},
-    'merge flash 65% + isolate': {
-        'type': 'isolate', 'merging': 'flash 65%', 'threads': THREADS,
-        'memory': MEMORY, 'k-mers': KMERS}}
+    'Tell-Seq meta + no merge': {
+        'type': 'meta', 'merging': 'no merge', 'threads': THREADS}
+}
 
 spades_cmd = QiitaCommand(
-    "spades", "Isolate and Metagenomic processing via spades", spades,
+    "cloudSPAdes", "Assembly via cloudSPAdes", spades,
     req_params, opt_params, outputs, default_params)
 
 plugin.register_command(spades_cmd)

qp_spades/qp_spades.py

@@ -16,6 +16,7 @@
 # resources per job
 WALLTIME = '200:00:00'
 MAX_RUNNING = 8
+MEMORY = '128g'
 FINISH_WALLTIME = '10:00:00'
 FINISH_MEMORY = '4g'
 
@@ -24,7 +25,6 @@ def spades_to_array(directory, output_dir, prefix_to_name, url,
                     job_id, params):
     environment = environ["ENVIRONMENT"]
     ppn = params["threads"]
-    memory = params["memory"]
 
     # 1. create file list
     num_samples = len(prefix_to_name)
@@ -43,16 +43,16 @@ def spades_to_array(directory, output_dir, prefix_to_name, url,
 
     # 2. format main comand
     command = (
-        f'spades.py --{params["type"]} -t {ppn} -m {memory} '
-        f'-k {params["k-mers"]} -o $OUTDIR/$SNAME')
+        f'spades.py --{params["type"]} -m {MEMORY.replace("g", "")} '
+        f'-t {ppn} -o $OUTDIR/$SNAME')
     if params['merging'].startswith('flash '):
         # get read length quickly; note that we are going to assume
         # that (1) the forward and reverse are the same length and (2)
         # all file pairs have the same length so only calculate once
         fp = glob(join(directory, list(prefix_to_name)[0] + '*'))[0]
         std_out, std_err, return_value = system_call(
                 f'zcat -c {fp} | head -n 2')
-        if return_value != 0:
+        if return_value != 0 or std_err:
             error_msg = (f"Error uncompressing: {fp}\n"
                          f"Std out: {std_out}\nStd err: {std_err}\n")
             raise ValueError(error_msg)
@@ -70,10 +70,10 @@ def spades_to_array(directory, output_dir, prefix_to_name, url,
             # spades
             f'{command} '
             '--merge $OUTDIR/${SNAME}.extendedFrags.fastq '
-            '-1 $OUTDIR/${SNAME}.notCombined_1.fastq '
-            '-2 $OUTDIR/${SNAME}.notCombined_2.fastq')
+            '--gemcode1-1 $OUTDIR/${SNAME}.notCombined_1.fastq '
+            '--gemcode1-2 $OUTDIR/${SNAME}.notCombined_2.fastq')
     else:
-        command = '%s -1 ${FWD} -2 ${REV}' % command
+        command = '%s --gemcode1-1 ${FWD} --gemcode1-2 ${REV}' % command
 
     # 3. create command for array submission
     marray = [
@@ -84,7 +84,7 @@ def spades_to_array(directory, output_dir, prefix_to_name, url,
         '#SBATCH -N 1',
         f'#SBATCH -n {ppn}',
         f'#SBATCH --time {WALLTIME}',
-        f'#SBATCH --mem {memory}g',
+        f'#SBATCH --mem {MEMORY}',
         f'#SBATCH --output {output_dir}/{job_id}_%a.log',
         f'#SBATCH --error {output_dir}/{job_id}_%a.err',
         f'#SBATCH --array 1-{num_samples}%{MAX_RUNNING}',
@@ -107,7 +107,7 @@ def spades_to_array(directory, output_dir, prefix_to_name, url,
         '#!/bin/bash',
         '#SBATCH -p qiita',
         '#SBATCH --mail-user [email protected]',
-        f'#SBATCH --job-name merge-{job_id}',
+        f'#SBATCH --job-name finish-{job_id}',
         '#SBATCH -N 1',
         '#SBATCH -n 1',
         f'#SBATCH --time {FINISH_WALLTIME}',
@@ -164,23 +164,15 @@ def spades(qclient, job_id, parameters, out_dir):
     files, prep = qclient.artifact_and_preparation_files(artifact_id)
     prep_info = prep.set_index('run_prefix')['sample_name'].to_dict()
 
-    missing = []
     outfiles = []
     for run_prefix, sname in prep_info.items():
         scaffold = join(out_dir, run_prefix, 'scaffolds.fasta')
+        new_scaffold = join(out_dir, run_prefix, f'{run_prefix}.fasta')
         if exists(scaffold):
-            new_scaffold = join(out_dir, run_prefix, f'{run_prefix}.fasta')
             run(['mv', scaffold, new_scaffold], stdout=PIPE)
-            outfiles.append((new_scaffold, 'preprocessed_fasta'))
         else:
-            missing.append(f'{sname} [{run_prefix}]')
-
-    if missing:
-        error_msg = (
-            'There was no scaffolds.fasta for samples: %s. Contact: '
-            '[email protected] and add this job id: %s' % (
-                ', '.join(missing), job_id))
-        return False, None, error_msg
+            run(['touch', new_scaffold], stdout=PIPE)
+        outfiles.append((new_scaffold, 'preprocessed_fasta'))
 
     # Step 4 generating artifacts
     msg = "Step 4 of 4: Generating new artifact"

qp_spades/tests/test_spades.py

@@ -89,7 +89,7 @@ def test_spades_to_array(self):
         # testing isolate/no-merge
         params = {
             'type': 'isolate', 'merging': 'no merge', 'input': self.aid,
-            'threads': 5, 'memory': 200, 'k-mers': '21,33,55,77,99,127'}
+            'threads': 5}
         out_dir = mkdtemp()
         self._clean_up_files.append(out_dir)
         files, prep = self.qclient.artifact_and_preparation_files(self.aid)
@@ -121,7 +121,7 @@ def test_spades_to_array(self):
         # can reuse
         params = {
             'type': 'meta', 'merging': 'flash 65%', 'input': self.aid,
-            'threads': 5, 'memory': 200, 'k-mers': '21,33,55,77,99,127'}
+            'threads': 5}
         prefix_to_name = prep.set_index('run_prefix')['sample_name'].to_dict()
 
         main_fp, finish_fp = spades_to_array(
@@ -159,24 +159,15 @@ def test_spades(self):
         # testing error
         params = {
             'type': 'meta', 'merging': 'flash 65%', 'input': self.aid,
-            'threads': 5, 'memory': 200, 'k-mers': '21,33,55,77,99,127'}
+            'threads': 5}
         self.data['command'] = dumps(
-            [plugin_details['name'], plugin_details['version'], 'spades'])
+            [plugin_details['name'], plugin_details['version'], 'cloudSPAdes'])
         self.data['parameters'] = dumps(params)
         jid = self.qclient.post(
             '/apitest/processing_job/', data=self.data)['job']
 
         out_dir = mkdtemp()
         self._clean_up_files.append(out_dir)
-        success, artifact, msg = spades(self.qclient, jid, params, out_dir)
-        self.assertIsNone(artifact)
-        self.assertFalse(success)
-        self.assertEqual(msg, f'There was no scaffolds.fasta for samples: '
-                         '1.SKB8.640193 [S22205_S104], 1.SKD8.640184 '
-                         '[S22282_S102]. Contact: [email protected] and '
-                         f'add this job id: {jid}')
-
-        # testing success
         mkdir(f'{out_dir}/S22205_S104')
         mkdir(f'{out_dir}/S22282_S102')
         Path(f'{out_dir}/S22205_S104/scaffolds.fasta').touch()
@@ -196,9 +187,9 @@ def test_spades(self):
 #SBATCH --mail-user [email protected]
 #SBATCH --job-name qiita_job_id
 #SBATCH -N 1
-#SBATCH -n 5
+#SBATCH -n {threads}
 #SBATCH --time 200:00:00
-#SBATCH --mem 200g
+#SBATCH --mem 128g
 #SBATCH --output {out_dir}/qiita_job_id_%a.log
 #SBATCH --error {out_dir}/qiita_job_id_%a.err
 #SBATCH --array 1-2%8
@@ -213,15 +204,15 @@ def test_spades(self):
 FWD=$(echo -e $args | awk '{{ print $1 }}')
 REV=$(echo -e $args | awk '{{ print $2 }}')
 SNAME=$(echo -e $args | awk '{{ print $3 }}')
-spades.py --{type} -t {threads} -m {memory} -k {k-mers} -o $OUTDIR/$SNAME \
--1 ${{FWD}} -2 ${{REV}}
+spades.py --{type} -m 128 -t {threads} -o $OUTDIR/$SNAME \
+--gemcode1-1 ${{FWD}} --gemcode1-2 ${{REV}}
 date
 """
 
 EXP_FINISH = """#!/bin/bash
 #SBATCH -p qiita
 #SBATCH --mail-user [email protected]
-#SBATCH --job-name merge-qiita_job_id
+#SBATCH --job-name finish-qiita_job_id
 #SBATCH -N 1
 #SBATCH -n 1
 #SBATCH --time 10:00:00
@@ -248,9 +239,9 @@ def test_spades(self):
 #SBATCH --mail-user [email protected]
 #SBATCH --job-name qiita_job_id
 #SBATCH -N 1
-#SBATCH -n 5
+#SBATCH -n {threads}
 #SBATCH --time 200:00:00
-#SBATCH --mem 200g
+#SBATCH --mem 128g
 #SBATCH --output {out_dir}/qiita_job_id_%a.log
 #SBATCH --error {out_dir}/qiita_job_id_%a.err
 #SBATCH --array 1-2%8
@@ -267,11 +258,10 @@ def test_spades(self):
 SNAME=$(echo -e $args | awk '{{ print $3 }}')
 flash --threads {threads} --max-overlap=97 --output-directory $OUTDIR \
 --output-prefix="$SNAME" ${{FWD}} ${{REV}} --max-mismatch-density=0.1 \
-> $OUTDIR/${{SNAME}}.flash.log 2>&1 && spades.py --{type} -t {threads} \
--m {memory} -k {k-mers} -o $OUTDIR/$SNAME \
---merge $OUTDIR/${{SNAME}}.extendedFrags.fastq \
--1 $OUTDIR/${{SNAME}}.notCombined_1.fastq \
--2 $OUTDIR/${{SNAME}}.notCombined_2.fastq
+> $OUTDIR/${{SNAME}}.flash.log 2>&1 && spades.py --{type} -m 128 -t \
+{threads} -o $OUTDIR/$SNAME --merge $OUTDIR/${{SNAME}}.extendedFrags.fastq \
+--gemcode1-1 $OUTDIR/${{SNAME}}.notCombined_1.fastq \
+--gemcode1-2 $OUTDIR/${{SNAME}}.notCombined_2.fastq
 date
 """