Commit 6c95dce1 authored by Jarrod's avatar Jarrod 💬

Update index.js

parent 845a0a46
......@@ -7,7 +7,7 @@ class Tokenizer {
this.options = options
}
tokenize(text) {
tokenize(text, allowSingleMultiCharWords) {
let result = []
let i = 0
......@@ -48,7 +48,7 @@ class Tokenizer {
}
// JAL - Added IF
if (entries[j]['simplified'] != text && entries[j]['traditional'] != text) {
if (allowSingleMultiCharWords || (entries[j]['simplified'] != text && entries[j]['traditional'] != text)) {
entry = entries[j]
pushEntry(entry['traditional'])
break
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment