From b40160731559fe722f811924835713ed25901a1b Mon Sep 17 00:00:00 2001
From: M66B <M66B@users.noreply.github.com>
Date: Thu, 29 Sep 2022 13:29:38 +0200
Subject: [PATCH] Use break iterator for text indexing

---
 .../java/eu/faircode/email/Fts4DbHelper.java  | 33 ++++++++++++++++---
 1 file changed, 29 insertions(+), 4 deletions(-)

diff --git a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
index 58e2f555ca..7c7eed410b 100644
--- a/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
+++ b/app/src/main/java/eu/faircode/email/Fts4DbHelper.java
@@ -25,6 +25,7 @@ import android.content.Context;
 import android.database.Cursor;
 import android.database.sqlite.SQLiteDatabase;
 import android.database.sqlite.SQLiteOpenHelper;
+import android.os.Build;
 import android.text.TextUtils;
 
 import java.io.File;
@@ -117,10 +118,10 @@ public class Fts4DbHelper extends SQLiteOpenHelper {
         cv.put("folder", message.folder);
         cv.put("time", message.received);
         cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
-        cv.put("subject", message.subject == null ? "" : message.subject);
-        cv.put("keyword", TextUtils.join(", ", message.keywords));
-        cv.put("text", text);
-        cv.put("notes", message.notes);
+        cv.put("subject", breakText(message.subject));
+        cv.put("keyword", TextUtils.join(" ", message.keywords));
+        cv.put("text", breakText(text));
+        cv.put("notes", breakText(message.notes));
         db.insertWithOnConflict("message", null, cv, SQLiteDatabase.CONFLICT_FAIL);
     }
 
@@ -132,6 +133,30 @@ public class Fts4DbHelper extends SQLiteOpenHelper {
         db.delete("message", "rowid = ?", new String[]{Long.toString(id)});
     }
 
+    static String breakText(String text) {
+        if (TextUtils.isEmpty(text))
+            return "";
+
+        if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N)
+            return text;
+
+        StringBuilder sb = new StringBuilder();
+        android.icu.text.BreakIterator boundary = android.icu.text.BreakIterator.getWordInstance();
+        boundary.setText(text);
+        int start = boundary.first();
+        for (int end = boundary.next(); end != android.icu.text.BreakIterator.DONE; end = boundary.next()) {
+            String word = text.substring(start, end).trim();
+            if (!TextUtils.isEmpty(word)) {
+                if (sb.length() > 0)
+                    sb.append(' ');
+                sb.append(word);
+            }
+            start = end;
+        }
+
+        return sb.toString();
+    }
+
     static List<String> getSuggestions(SQLiteDatabase db, String query, int max) {
         List<String> result = new ArrayList<>();