mirror
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Added FTS index worker

This commit is contained in:
M66B 2020-01-14 22:39:35 +01:00
parent acca352e5a
commit 9c9a96e8f1
9 changed files with 2233 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2066
app/schemas/eu.faircode.email.DB/130.json Normal file
View File

File diff suppressed because it is too large Load Diff

6
app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java
View File

@ -65,6 +65,8 @@ import javax.mail.search.RecipientStringTerm;
import javax.mail.search.SearchTerm;
import javax.mail.search.SubjectTerm;
import io.requery.android.database.sqlite.SQLiteDatabase;
public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMessageEx> {
private Context context;
private Long folder;
@ -189,7 +191,9 @@ public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMe
(find == null || !find.startsWith(context.getString(R.string.title_search_special_prefix) + ":"))) {
if (state.ids == null) {
FtsDbHelper ftsDb = new FtsDbHelper(context);
state.ids = ftsDb.match(query);
try (SQLiteDatabase sdb = ftsDb.getReadableDatabase()) {
state.ids = ftsDb.match(sdb, folder, query);
}
}
try {

16
app/src/main/java/eu/faircode/email/Core.java
View File

@ -959,6 +959,8 @@ class Core {
int count = ifolder.getMessageCount();
db.folder().setFolderTotal(folder.id, count < 0 ? null : count);
}
WorkerFts.init(context);
}
private static void onDelete(Context context, JSONArray jargs, EntityFolder folder, EntityMessage message, IMAPFolder ifolder) throws MessagingException {
@ -1986,6 +1988,8 @@ class Core {
Log.i(folder.name + " end sync state=" + state);
db.folder().setFolderSyncState(folder.id, null);
}
WorkerFts.init(context);
}
static EntityMessage synchronizeMessage(
@ -2240,12 +2244,6 @@ class Core {
Log.i(folder.name + " inline downloaded message id=" + message.id +
" size=" + message.size + "/" + (body == null ? null : body.length()));
boolean fts = prefs.getBoolean("fts", false);
if (fts) {
FtsDbHelper ftsDb = new FtsDbHelper(context);
ftsDb.insert(message, HtmlHelper.getText(body));
}
Long size = parts.getBodySize();
if (TextUtils.isEmpty(body) && size != null && size > 0)
reportEmptyMessage(context, account, istore);
@ -2573,12 +2571,6 @@ class Core {
Log.i(folder.name + " downloaded message id=" + message.id +
" size=" + message.size + "/" + (body == null ? null : body.length()));
boolean fts = prefs.getBoolean("fts", false);
if (fts) {
FtsDbHelper ftsDb = new FtsDbHelper(context);
ftsDb.insert(message, HtmlHelper.getText(body));
}
Long size = parts.getBodySize();
if (TextUtils.isEmpty(body) && size != null && size > 0)
reportEmptyMessage(context, account, istore);

9
app/src/main/java/eu/faircode/email/DB.java
View File

@ -56,7 +56,7 @@ import io.requery.android.database.sqlite.RequerySQLiteOpenHelperFactory;
// https://developer.android.com/topic/libraries/architecture/room.html
@Database(
version = 129,
version = 130,
entities = {
EntityIdentity.class,
EntityAccount.class,
@ -1247,6 +1247,13 @@ public abstract class DB extends RoomDatabase {
db.execSQL("ALTER TABLE `account` ADD COLUMN `poll_exempted` INTEGER NOT NULL DEFAULT 0");
}
})
.addMigrations(new Migration(129, 130) {
@Override
public void migrate(@NonNull SupportSQLiteDatabase db) {
Log.i("DB migration from version " + startVersion + " to " + endVersion);
db.execSQL("ALTER TABLE `message` ADD COLUMN `fts` INTEGER NOT NULL DEFAULT 0");
}
})
.build();
}

8
app/src/main/java/eu/faircode/email/DaoMessage.java
View File

@ -227,6 +227,11 @@ public interface DaoMessage {
" ORDER BY message.received DESC")
List<Long> getMessageIdsByFolder(Long folder);
@Query("SELECT id FROM message" +
" WHERE content AND NOT fts" +
" ORDER BY message.received DESC")
Cursor getMessageFts();
@Query("SELECT id, account, thread, (:find IS NULL" +
" OR `from` LIKE :find COLLATE NOCASE" +
" OR `to` LIKE :find COLLATE NOCASE" +
@ -437,6 +442,9 @@ public interface DaoMessage {
@Query("UPDATE message SET notifying = :notifying WHERE id = :id")
int setMessageNotifying(long id, int notifying);
@Query("UPDATE message SET fts = :fts WHERE id = :id")
int setMessageFts(long id, boolean fts);
@Query("UPDATE message SET received = :received WHERE id = :id")
int setMessageReceived(long id, long received);

2
app/src/main/java/eu/faircode/email/EntityMessage.java
View File

@ -154,6 +154,8 @@ public class EntityMessage implements Serializable {
@NonNull
public Integer notifying = 0;
@NonNull
public Boolean fts = false;
@NonNull
public Boolean ui_seen = false;
@NonNull
public Boolean ui_answered = false;

1
app/src/main/java/eu/faircode/email/FragmentOptionsMisc.java
View File

@ -132,6 +132,7 @@ public class FragmentOptionsMisc extends FragmentBase implements SharedPreferenc
@Override
public void onCheckedChanged(CompoundButton compoundButton, boolean checked) {
prefs.edit().putBoolean("fts", checked).apply();
WorkerFts.init(getContext());
}
});

56
app/src/main/java/eu/faircode/email/FtsDbHelper.java
View File

@ -45,7 +45,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
@Override
public void onCreate(SQLiteDatabase db) {
Log.i("FTS create");
db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`time`, `address`, `subject`, `keyword`, `text`)");
db.execSQL("CREATE VIRTUAL TABLE `message` USING fts4(`folder`, `time`, `address`, `subject`, `keyword`, `text`)");
}
@Override
@ -53,7 +53,7 @@ public class FtsDbHelper extends SQLiteOpenHelper {
// Do nothing
}
void insert(EntityMessage message, String text) {
void insert(SQLiteDatabase db, EntityMessage message, String text) {
Log.i("FTS insert id=" + message.id + " subject=" + message.subject + " text=" + text);
List<Address> address = new ArrayList<>();
if (message.from != null)
@ -63,39 +63,39 @@ public class FtsDbHelper extends SQLiteOpenHelper {
if (message.cc != null)
address.addAll(Arrays.asList(message.cc));
try (SQLiteDatabase db = getWritableDatabase()) {
try {
db.beginTransaction();
try {
db.beginTransaction();
db.delete("message", "docid = ?", new Object[]{message.id});
db.delete("message", "docid = ?", new Object[]{message.id});
ContentValues cv = new ContentValues();
cv.put("docid", message.id);
cv.put("time", message.received);
cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
cv.put("subject", message.subject == null ? "" : message.subject);
cv.put("keyword", TextUtils.join(", ", message.keywords));
cv.put("text", text);
db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
ContentValues cv = new ContentValues();
cv.put("docid", message.id);
cv.put("folder", message.folder);
cv.put("time", message.received);
cv.put("address", MessageHelper.formatAddresses(address.toArray(new Address[0]), true, false));
cv.put("subject", message.subject == null ? "" : message.subject);
cv.put("keyword", TextUtils.join(", ", message.keywords));
cv.put("text", text);
db.insert("message", SQLiteDatabase.CONFLICT_FAIL, cv);
db.setTransactionSuccessful();
} finally {
db.endTransaction();
}
db.setTransactionSuccessful();
} catch (Throwable ex) {
Log.e(ex);
} finally {
db.endTransaction();
}
}
List<Long> match(String search) {
Log.i("FTS search=" + search);
List<Long> match(SQLiteDatabase db, Long folder, String search) {
Log.i("FTS folder=" + folder + " search=" + search);
List<Long> result = new ArrayList<>();
try (SQLiteDatabase db = getReadableDatabase()) {
try (Cursor cursor = db.query(
"message", new String[]{"docid"},
"message MATCH ?", new Object[]{search},
null, null, "time DESC", null)) {
while (cursor != null && cursor.moveToNext())
result.add(cursor.getLong(0));
}
try (Cursor cursor = db.query(
"message", new String[]{"docid"},
folder == null ? "message MATCH ?" : "folder = ? AND message MATCH ?",
folder == null ? new Object[]{search} : new Object[]{folder, search},
null, null, "time DESC", null)) {
while (cursor != null && cursor.moveToNext())
result.add(cursor.getLong(0));
}
Log.i("FTS result=" + result.size());
return result;

111
app/src/main/java/eu/faircode/email/WorkerFts.java Normal file
View File

@ -0,0 +1,111 @@
package eu.faircode.email;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see <http://www.gnu.org/licenses/>.
Copyright 2018-2020 by Marcel Bokhorst (M66B)
*/
import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import androidx.annotation.NonNull;
import androidx.preference.PreferenceManager;
import androidx.work.ExistingWorkPolicy;
import androidx.work.OneTimeWorkRequest;
import androidx.work.WorkManager;
import androidx.work.Worker;
import androidx.work.WorkerParameters;
import java.io.File;
import java.util.concurrent.TimeUnit;
import io.requery.android.database.sqlite.SQLiteDatabase;
public class WorkerFts extends Worker {
public WorkerFts(@NonNull Context context, @NonNull WorkerParameters workerParams) {
super(context, workerParams);
Log.i("Instance " + getName());
}
@NonNull
@Override
public Result doWork() {
try {
Log.i("FTS index");
int indexed = 0;
FtsDbHelper ftsDb = new FtsDbHelper(getApplicationContext());
try (SQLiteDatabase sdb = ftsDb.getWritableDatabase()) {
DB db = DB.getInstance(getApplicationContext());
try (Cursor cursor = db.message().getMessageFts()) {
while (cursor.moveToNext()) {
EntityMessage message = db.message().getMessage(cursor.getLong(0));
if (message != null)
try {
Log.i("FTS index=" + message.id);
File file = message.getFile(getApplicationContext());
String html = Helper.readText(file);
ftsDb.insert(sdb, message, HtmlHelper.getText(html));
db.message().setMessageFts(message.id, true);
indexed++;
} catch (Throwable ex) {
Log.e(ex);
}
}
}
}
Log.i("FTS indexed=" + indexed);
return Result.success();
} catch (Throwable ex) {
Log.e(ex);
return Result.failure();
}
}
static void init(Context context) {
try {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean fts = prefs.getBoolean("fts", true);
if (fts) {
Log.i("Queuing " + getName());
OneTimeWorkRequest workRequest =
new OneTimeWorkRequest.Builder(WorkerFts.class)
.setInitialDelay(30, TimeUnit.SECONDS)
.build();
WorkManager.getInstance(context)
.enqueueUniqueWork(getName(), ExistingWorkPolicy.REPLACE, workRequest);
Log.i("Queued " + getName());
} else {
Log.i("Cancelling " + getName());
WorkManager.getInstance(context).cancelUniqueWork(getName());
Log.i("Cancelled " + getName());
}
} catch (IllegalStateException ex) {
// https://issuetracker.google.com/issues/138465476
Log.w(ex);
}
}
private static String getName() {
return WorkerFts.class.getSimpleName();
}
}