Data using simple preprocessing

2020-05-03 01:28:33 +02:00 · 2020-05-03 01:28:33 +02:00 · b5f573f927
parent 68e5f235a1
commit b5f573f927
2 changed files with 504143 additions and 0 deletions
--- a/data/eu_train_simple.tsv
+++ b/data/eu_train_simple.tsv
--- a/preprocessing/preprocess-simple.sh
+++ b/preprocessing/preprocess-simple.sh
@ -0,0 +1,4 @@
+#!/usr/bin/env sh
+
+sed -E 's|^--?||;s|^"?–||;s|—||;s|―||;s|^_||;s|^ ||' < "${1:-/dev/stdin}" | 
+    uniq | paste - -