mirror
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Switch to sqlite/fts4

This commit is contained in:
M66B 2022-09-26 08:13:03 +02:00
parent c4ea93f446
commit c30a1c60fe
7 changed files with 286 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
View File

@ -282,8 +282,8 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
if (criteria.fts && criteria.query != null) {
if (state.ids == null) {
SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
state.ids = FtsDbHelper.match(sdb, account, folder, exclude, criteria);
SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria);
EntityLog.log(context, "Boundary FTS " +
" account=" + account +
" folder=" + folder +

4
app/src/main/java/eu/faircode/email/FragmentDialogSearch.java
View File

@ -154,8 +154,8 @@ public class FragmentDialogSearch extends FragmentDialogBase {
return cursor;
if (cbSearchIndex.isEnabled() && cbSearchIndex.isChecked()) {
SQLiteDatabase db = FtsDbHelper.getInstance(context);
List<String> suggestions = FtsDbHelper.getSuggestions(
SQLiteDatabase db = Fts4DbHelper.getInstance(context);
List<String> suggestions = Fts4DbHelper.getSuggestions(
db,
typed + "%",
MAX_SUGGESTIONS);

10
app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java
View File

@ -502,12 +502,12 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
@Override
protected Void onExecute(Context context, Bundle args) {
try {
SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
FtsDbHelper.delete(sdb);
FtsDbHelper.optimize(sdb);
SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
Fts4DbHelper.delete(sdb);
Fts4DbHelper.optimize(sdb);
} catch (SQLiteDatabaseCorruptException ex) {
Log.e(ex);
FtsDbHelper.delete(context);
Fts4DbHelper.delete(context);
}
DB db = DB.getInstance(context);
@ -1762,7 +1762,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
tvFtsIndexed.setText(getString(R.string.title_advanced_fts_indexed,
stats.fts,
stats.total,
Helper.humanReadableByteCount(FtsDbHelper.size(tvFtsIndexed.getContext()))));
Helper.humanReadableByteCount(Fts4DbHelper.size(tvFtsIndexed.getContext()))));
last = stats;
}
});

267
app/src/main/java/eu/faircode/email/Fts4DbHelper.java Normal file
View File

@ -0,0 +1,267 @@
package eu.faircode.email;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see <http://www.gnu.org/licenses/>.
Copyright 2018-2022 by Marcel Bokhorst (M66B)
*/
import android.annotation.SuppressLint;
import android.content.ContentValues;
import android.content.Context;
import android.database.Cursor;
import android.text.TextUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import javax.mail.Address;
import io.requery.android.database.sqlite.SQLiteDatabase;
import io.requery.android.database.sqlite.SQLiteOpenHelper;
// https://www.sqlite.org/fts3.html
// fts4 requires sqlite version 3.7.4, API 21 Android
public class Fts4DbHelper extends SQLiteOpenHelper {
private Context context;
@SuppressLint("StaticFieldLeak")
private static Fts4DbHelper instance = null;
private static final int DATABASE_VERSION = 1;
private static final String DATABASE_NAME = "fts4.db";
private Fts4DbHelper(Context context) {
super(context.getApplicationContext(), DATABASE_NAME, null, DATABASE_VERSION);
this.context = context.getApplicationContext();
}
static SQLiteDatabase getInstance(Context context) {
if (instance == null) {
if (!context.getDatabasePath(DATABASE_NAME).exists()) {
Fts5DbHelper.delete(context);
DB.getInstance(context).message().resetFts();
}
instance = new Fts4DbHelper(context);
}
return instance.getWritableDatabase();
}
@Override
public void onCreate(SQLiteDatabase db) {
Log.i("FTS create");
db.execSQL("CREATE VIRTUAL TABLE `message`" +
" USING fts4" +
" (`account`" +
", `folder`" +
", `time`" +
", `address`" +
", `subject`" +
", `keyword`" +
", `text`" +
", `notes`" +
", notindexed=`account`" +
", notindexed=`folder`" +
", notindexed=`time`" +
", tokenize=unicode61 \"remove_diacritics=2\")");
// https://www.sqlite.org/fts3.html#tokenizer
// https://unicode.org/reports/tr29/
// https://www.sqlite.org/fts3.html#fts4aux
db.execSQL("CREATE VIRTUAL TABLE message_terms USING fts4aux('message');");
}
@Override
public void onUpgrade(SQLiteDatabase db, int oldVersion, int newVersion) {
Log.i("FTS upgrade from " + oldVersion + " to " + newVersion);
db.execSQL("DROP TABLE IF EXISTS `message`");
db.execSQL("DROP TABLE IF EXISTS `message_terms`");
onCreate(db);
DB.getInstance(context).message().resetFts();
}
static void insert(SQLiteDatabase db, EntityMessage message, String text) {
Log.i("FTS insert id=" + message.id);
List<Address> address = new ArrayList<>();
if (message.from != null)
address.addAll(Arrays.asList(message.from));
if (message.to != null)
address.addAll(Arrays.asList(message.to));
if (message.cc != null)
address.addAll(Arrays.asList(message.cc));
if (message.bcc != null)
address.addAll(Arrays.asList(message.bcc));
delete(db, message.id);
ContentValues cv = new ContentValues();
cv.put("rowid", message.id);
cv.put("account", message.account);
cv.put("folder", message.folder);
cv.put("time", message.received);
cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
cv.put("subject", message.subject == null ? "" : message.subject);
cv.put("keyword", TextUtils.join(", ", message.keywords));
cv.put("text", text);
cv.put("notes", message.notes);
db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
}
static void delete(SQLiteDatabase db) {
db.delete("message", null, null);
}
static void delete(SQLiteDatabase db, long id) {
db.delete("message", "rowid = ?", new Object[]{id});
}
static List<String> getSuggestions(SQLiteDatabase db, String query, int max) {
List<String> result = new ArrayList<>();
try (Cursor cursor = db.query(
"SELECT term FROM message_terms" +
" WHERE term LIKE ?" +
" ORDER BY occurrences DESC" +
" LIMIT " + max,
new Object[]{query})) {
while (cursor != null && cursor.moveToNext())
result.add(cursor.getString(0));
}
return result;
}
static List<Long> match(
SQLiteDatabase db,
Long account, Long folder, long[] exclude,
BoundaryCallbackMessages.SearchCriteria criteria) {
List<String> word = new ArrayList<>();
List<String> plus = new ArrayList<>();
List<String> minus = new ArrayList<>();
List<String> opt = new ArrayList<>();
StringBuilder all = new StringBuilder();
for (String w : criteria.query.trim().split("\\s+")) {
if (all.length() > 0)
all.append(' ');
if (w.length() > 1 && w.startsWith("+")) {
plus.add(w.substring(1));
all.append(w.substring(1));
} else if (w.length() > 1 && w.startsWith("-")) {
minus.add(w.substring(1));
all.append(w.substring(1));
} else if (w.length() > 1 && w.startsWith("?")) {
opt.add(w.substring(1));
all.append(w.substring(1));
} else {
word.add(w);
all.append(w);
}
}
StringBuilder sb = new StringBuilder();
if (plus.size() + minus.size() + opt.size() > 0) {
if (word.size() > 0)
sb.append(escape(TextUtils.join(" ", word)));
for (String p : plus) {
if (sb.length() > 0)
sb.append(" AND ");
sb.append(escape(p));
}
for (String m : minus) {
if (sb.length() > 0)
sb.append(" NOT ");
sb.append(escape(m));
}
if (sb.length() > 0) {
sb.insert(0, '(');
sb.append(')');
}
for (String o : opt) {
if (sb.length() > 0)
sb.append(" OR ");
sb.append(escape(o));
}
}
String search = (sb.length() > 0 ? sb.toString() : escape(criteria.query));
String select = "";
if (account != null)
select += "account = " + account + " AND ";
if (folder != null)
select += "folder = " + folder + " AND ";
if (exclude.length > 0) {
select += "NOT folder IN (";
for (int i = 0; i < exclude.length; i++) {
if (i > 0)
select += ", ";
select += exclude[i];
}
select += ") AND ";
}
if (criteria.after != null)
select += "time > " + criteria.after + " AND ";
if (criteria.before != null)
select += "time < " + criteria.before + " AND ";
Log.i("FTS select=" + select + " search=" + search);
List<Long> result = new ArrayList<>();
try (Cursor cursor = db.query(
"message", new String[]{"rowid"},
select + "message MATCH ?",
new Object[]{search},
null, null, "time DESC", null)) {
while (cursor != null && cursor.moveToNext())
result.add(cursor.getLong(0));
}
Log.i("FTS result=" + result.size());
return result;
}
private static String escape(String word) {
return "\"" + word.replaceAll("\"", "\"\"") + "\"";
}
static Cursor getIds(SQLiteDatabase db) {
return db.query(
"message", new String[]{"rowid"},
null, null,
null, null, "time");
}
static long size(Context context) {
return context.getDatabasePath(DATABASE_NAME).length();
}
static void optimize(SQLiteDatabase db) {
Log.i("FTS optimize");
db.execSQL("INSERT INTO message (message) VALUES ('optimize')");
}
static void delete(Context context) {
context.getDatabasePath(DATABASE_NAME).delete();
}
}

8
app/src/main/java/eu/faircode/email/FtsDbHelper.java → app/src/main/java/eu/faircode/email/Fts5DbHelper.java
View File

@ -35,23 +35,23 @@ import io.requery.android.database.sqlite.SQLiteDatabase;
import io.requery.android.database.sqlite.SQLiteOpenHelper;
// https://www.sqlite.org/fts5.html
public class FtsDbHelper extends SQLiteOpenHelper {
public class Fts5DbHelper extends SQLiteOpenHelper {
private Context context;
@SuppressLint("StaticFieldLeak")
private static FtsDbHelper instance = null;
private static Fts5DbHelper instance = null;
private static final int DATABASE_VERSION = 5;
private static final String DATABASE_NAME = "fts.db";
private FtsDbHelper(Context context) {
private Fts5DbHelper(Context context) {
super(context.getApplicationContext(), DATABASE_NAME, null, DATABASE_VERSION);
this.context = context.getApplicationContext();
}
static SQLiteDatabase getInstance(Context context) {
if (instance == null)
instance = new FtsDbHelper(context);
instance = new Fts5DbHelper(context);
return instance.getWritableDatabase();
}

8
app/src/main/java/eu/faircode/email/WorkerCleanup.java
View File

@ -334,21 +334,21 @@ public class WorkerCleanup extends Worker {
Log.i("Cleanup FTS=" + fts);
if (fts) {
int deleted = 0;
SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
try (Cursor cursor = FtsDbHelper.getIds(sdb)) {
SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
try (Cursor cursor = Fts4DbHelper.getIds(sdb)) {
while (cursor.moveToNext()) {
long rowid = cursor.getLong(0);
EntityMessage message = db.message().getMessage(rowid);
if (message == null || !message.fts) {
Log.i("Deleting FTS rowid=" + rowid);
FtsDbHelper.delete(sdb, rowid);
Fts4DbHelper.delete(sdb, rowid);
deleted++;
}
}
}
Log.i("Cleanup FTS=" + deleted);
if (manual)
FtsDbHelper.optimize(sdb);
Fts4DbHelper.optimize(sdb);
}
Log.i("Cleanup contacts");

4
app/src/main/java/eu/faircode/email/WorkerFts.java
View File

@ -65,7 +65,7 @@ public class WorkerFts extends Worker {
List<Long> ids = new ArrayList<>(INDEX_BATCH_SIZE);
DB db = DB.getInstance(context);
SQLiteDatabase sdb = FtsDbHelper.getInstance(context);
SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
try (Cursor cursor = db.message().getMessageFts()) {
while (cursor != null && cursor.moveToNext())
@ -92,7 +92,7 @@ public class WorkerFts extends Worker {
try {
sdb.beginTransaction();
FtsDbHelper.insert(sdb, message, text);
Fts4DbHelper.insert(sdb, message, text);
sdb.setTransactionSuccessful();
} finally {
sdb.endTransaction();