If a token is completely erased by the MWT seq2seq, restore it as a single word. #1401

Author: AngledLuffa

stanza/models/mwt/trainer.py

@@ -87,6 +87,10 @@ def predict(self, batch, unsort=True):
                     pred_tokens.append("".join(pred_seq))
         else:
             pred_tokens = ["".join(seq) for seq in pred_seqs] # join chars to be tokens
+            # if any tokens are predicted to expand to blank,
+            # that is likely an error.  use the original text
+            # this originally came up with the Spanish model turning 's' into a blank
+            pred_tokens = [x if x else y for x, y in zip(pred_tokens, orig_text)]
         if unsort:
             pred_tokens = utils.unsort(pred_tokens, orig_idx)
         return pred_tokens