Add symbols and punctuation to tokenizer

a3ba79a6 · Megan Henning · 37399486 · a3ba79a6
Commit a3ba79a6 authored 7 years ago by Megan Henning
--- a/bin/load_mappings.py
+++ b/bin/load_mappings.py
@@ -30,7 +30,9 @@ ANALYSIS = {
            "max_gram": 100,
            "token_chars": [
                "letter",
-                "digit"
+                "digit",
+                "symbol",
+                "punctuation"
            ]
        }
    }