We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent fcce71c commit 3ddcc89Copy full SHA for 3ddcc89
language_translation/src/data.py
@@ -42,14 +42,14 @@ def get_data(opts):
42
43
# Build a vocabulary object for these languages
44
src_vocab = build_vocab_from_iterator(
45
- _yield_tokens(train_iterator, src_tokenizer, src_lang),
+ _yield_tokens(train_iterator, src_tokenizer, True),
46
min_freq=1,
47
specials=list(special_symbols.keys()),
48
special_first=True
49
)
50
51
tgt_vocab = build_vocab_from_iterator(
52
- _yield_tokens(train_iterator, tgt_tokenizer, tgt_lang),
+ _yield_tokens(train_iterator, tgt_tokenizer, False),
53
54
55
0 commit comments