package eu.faircode.email;
import static eu.faircode.email.ServiceAuthenticator.AUTH_TYPE_PASSWORD;
import android.app.ActivityManager;
import android.content.ContentValues;
import android.content.Context;
import android.content.SharedPreferences;
import android.database.Cursor;
import android.database.sqlite.SQLiteDatabase;
import android.database.sqlite.SQLiteDatabaseCorruptException;
import android.net.Uri;
import android.text.TextUtils;
import androidx.annotation.NonNull;
import androidx.lifecycle.Observer;
import androidx.preference.PreferenceManager;
import androidx.room.Database;
import androidx.room.DatabaseConfiguration;
import androidx.room.InvalidationTracker;
import androidx.room.Room;
import androidx.room.RoomDatabase;
import androidx.room.TypeConverter;
import androidx.room.TypeConverters;
import androidx.room.migration.Migration;
import androidx.sqlite.db.SupportSQLiteDatabase;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.File;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ExecutorService;
import javax.mail.Address;
import javax.mail.internet.InternetAddress;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see .
Copyright 2018-2024 by Marcel Bokhorst (M66B)
*/
// https://developer.android.com/topic/libraries/architecture/room.html
@Database(
version = 291,
entities = {
EntityIdentity.class,
EntityAccount.class,
EntityFolder.class,
EntityMessage.class,
EntityAttachment.class,
EntityOperation.class,
EntityContact.class,
EntityCertificate.class,
EntityAnswer.class,
EntityRule.class,
EntitySearch.class,
EntityLog.class
},
views = {
TupleAccountView.class,
TupleIdentityView.class,
TupleFolderView.class
}
)
@TypeConverters({DB.Converters.class})
public abstract class DB extends RoomDatabase {
public abstract DaoAccount account();
public abstract DaoIdentity identity();
public abstract DaoFolder folder();
public abstract DaoMessage message();
public abstract DaoAttachment attachment();
public abstract DaoOperation operation();
public abstract DaoContact contact();
public abstract DaoCertificate certificate();
public abstract DaoAnswer answer();
public abstract DaoRule rule();
public abstract DaoSearch search();
public abstract DaoLog log();
private static int sPid;
private static Context sContext;
private static DB sInstance;
static final String DB_NAME = "fairemail";
static final int DEFAULT_QUERY_THREADS = 4; // AndroidX default thread count: 4
static final int DEFAULT_CACHE_SIZE = 20; // percentage of memory class
private static final int DB_JOURNAL_SIZE_LIMIT = 1048576; // requery/sqlite-android default
private static final int DB_CHECKPOINT = 1000; // requery/sqlite-android default
private static ExecutorService executor =
Helper.getBackgroundExecutor(0, "db");
private static final String[] DB_TABLES = new String[]{
"identity", "account", "folder", "message", "attachment", "operation", "contact", "certificate", "answer", "rule", "search", "log"};
private static final List DB_PRAGMAS = Collections.unmodifiableList(Arrays.asList(
"synchronous", "journal_mode", "busy_timeout",
"wal_checkpoint", "wal_autocheckpoint", "journal_size_limit",
"page_count", "page_size", "max_page_count", "freelist_count",
"cache_size", "cache_spill",
"soft_heap_limit", "hard_heap_limit", "mmap_size",
"foreign_keys", "auto_vacuum",
"recursive_triggers",
"compile_options"
));
@Override
public void init(@NonNull DatabaseConfiguration configuration) {
File dbfile = configuration.context.getDatabasePath(DB_NAME);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(configuration.context);
boolean sqlite_integrity_check = prefs.getBoolean("sqlite_integrity_check", true);
// https://www.sqlite.org/pragma.html#pragma_integrity_check
if (sqlite_integrity_check && dbfile.exists()) {
String check = (Helper.isRedmiNote() || Helper.isOnePlus() || Helper.isOppo() || BuildConfig.DEBUG
? "integrity_check" : "quick_check");
try (SQLiteDatabase db = SQLiteDatabase.openDatabase(dbfile.getPath(), null, SQLiteDatabase.OPEN_READWRITE)) {
Log.i("PRAGMA " + check);
try (Cursor cursor = db.rawQuery("PRAGMA " + check + ";", null)) {
while (cursor.moveToNext()) {
String line = cursor.getString(0);
if ("ok".equals(line))
Log.i("PRAGMA " + check + "=" + line);
else
Log.e("PRAGMA " + check + "=" + line);
}
}
} catch (SQLiteDatabaseCorruptException ex) {
Log.e(ex);
Helper.secureDelete(dbfile);
} catch (Throwable ex) {
Log.e(ex);
/*
java.lang.String, java.lang.String, android.os.Bundle)' on a null object reference
at android.provider.Settings$NameValueCache.getStringForUser(Settings.java:3002)
at android.provider.Settings$Global.getStringForUser(Settings.java:16253)
at android.provider.Settings$Global.getString(Settings.java:16241)
at android.database.sqlite.SQLiteCompatibilityWalFlags.initIfNeeded(SQLiteCompatibilityWalFlags.java:105)
at android.database.sqlite.SQLiteCompatibilityWalFlags.isLegacyCompatibilityWalEnabled(SQLiteCompatibilityWalFlags.java:57)
at android.database.sqlite.SQLiteDatabase.(SQLiteDatabase.java:321)
at android.database.sqlite.SQLiteDatabase.openDatabase(SQLiteDatabase.java:788)
at android.database.sqlite.SQLiteDatabase.openDatabase(SQLiteDatabase.java:737)
at eu.faircode.email.DB.init(SourceFile:61)
at androidx.room.RoomDatabase$Builder.build(SourceFile:274)
at eu.faircode.email.DB.getInstance(SourceFile:106)
at eu.faircode.email.DB.setupViewInvalidation(SourceFile:1)
at eu.faircode.email.ApplicationEx.onCreate(SourceFile:140)
at android.app.Instrumentation.callApplicationOnCreate(Instrumentation.java:1229)
at android.app.ActivityThread.handleBindApplication(ActivityThread.java:6719)
*/
}
}
// https://www.sqlite.org/pragma.html#pragma_wal_autocheckpoint
if (BuildConfig.DEBUG && dbfile.exists()) {
try (SQLiteDatabase db = SQLiteDatabase.openDatabase(dbfile.getPath(), null, SQLiteDatabase.OPEN_READWRITE)) {
Log.i("Set PRAGMA wal_autocheckpoint=" + DB_CHECKPOINT);
try (Cursor cursor = db.rawQuery("PRAGMA wal_autocheckpoint=" + DB_CHECKPOINT + ";", null)) {
cursor.moveToNext(); // required
}
} catch (Throwable ex) {
Log.e(ex);
}
}
super.init(configuration);
}
static void setupViewInvalidation(Context context) {
// This needs to be done on a foreground thread
DB db = DB.getInstance(context);
db.account().liveAccountView().observeForever(new Observer>() {
private List last = null;
@Override
public void onChanged(List accounts) {
if (accounts == null)
accounts = new ArrayList<>();
boolean changed = false;
if (last == null || last.size() != accounts.size())
changed = true;
else
for (int i = 0; i < accounts.size(); i++)
if (!accounts.get(i).equals(last.get(i))) {
changed = true;
last = accounts;
}
if (changed) {
Log.i("Invalidating account view");
last = accounts;
db.getInvalidationTracker().notifyObserversByTableNames("message");
}
}
});
db.identity().liveIdentityView().observeForever(new Observer>() {
private List last = null;
@Override
public void onChanged(List identities) {
if (identities == null)
identities = new ArrayList<>();
boolean changed = false;
if (last == null || last.size() != identities.size())
changed = true;
else
for (int i = 0; i < identities.size(); i++)
if (!identities.get(i).equals(last.get(i))) {
changed = true;
last = identities;
}
if (changed) {
Log.i("Invalidating identity view");
last = identities;
db.getInvalidationTracker().notifyObserversByTableNames("message");
}
}
});
db.folder().liveFolderView().observeForever(new Observer>() {
private List last = null;
@Override
public void onChanged(List folders) {
if (folders == null)
folders = new ArrayList<>();
boolean changed = false;
if (last == null || last.size() != folders.size())
changed = true;
else
for (int i = 0; i < folders.size(); i++)
if (!folders.get(i).equals(last.get(i))) {
changed = true;
last = folders;
}
if (changed) {
Log.i("Invalidating folder view");
last = folders;
db.getInvalidationTracker().notifyObserversByTableNames("account", "message");
}
}
});
}
static void createEmergencyBackup(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean emergency_file = prefs.getBoolean("emergency_file", true);
File emergency = new File(context.getFilesDir(), "emergency.json");
if (emergency_file) {
Log.i("Creating emergency backup");
try {
DB db = DB.getInstance(context);
JSONArray jaccounts = new JSONArray();
List accounts = db.account().getAccounts();
for (EntityAccount account : accounts) {
JSONObject jaccount = account.toJSON();
JSONArray jfolders = new JSONArray();
List folders = db.folder().getFolders(account.id, false, true);
for (EntityFolder folder : folders)
jfolders.put(folder.toJSON());
jaccount.put("folders", jfolders);
JSONArray jidentities = new JSONArray();
List identities = db.identity().getIdentities(account.id);
for (EntityIdentity identity : identities)
jidentities.put(identity.toJSON());
jaccount.put("identities", jidentities);
jaccounts.put(jaccount);
}
Helper.writeText(emergency, jaccounts.toString(2));
} catch (Throwable ex) {
Log.e(ex);
}
} else
Helper.secureDelete(emergency);
}
private static void checkEmergencyBackup(Context context) {
try {
File dbfile = context.getDatabasePath(DB_NAME);
if (dbfile.exists()) {
Log.i("Emergency restore /dbfile");
return;
}
File emergency = new File(context.getFilesDir(), "emergency.json");
if (!emergency.exists()) {
Log.i("Emergency restore /json");
return;
}
DB db = DB.getInstance(context);
if (db.account().getAccounts().size() > 0) {
Log.e("Emergency restore /accounts");
return;
}
Log.e("Emergency restore");
String json = Helper.readText(emergency);
JSONArray jaccounts = new JSONArray(json);
for (int a = 0; a < jaccounts.length(); a++) {
JSONObject jaccount = jaccounts.getJSONObject(a);
EntityAccount account = EntityAccount.fromJSON(jaccount);
account.created = new Date().getTime();
account.id = db.account().insertAccount(account);
JSONArray jfolders = jaccount.getJSONArray("folders");
for (int f = 0; f < jfolders.length(); f++) {
EntityFolder folder = EntityFolder.fromJSON(jfolders.getJSONObject(f));
folder.account = account.id;
db.folder().insertFolder(folder);
}
JSONArray jidentities = jaccount.getJSONArray("identities");
for (int i = 0; i < jidentities.length(); i++) {
EntityIdentity identity = EntityIdentity.fromJSON(jidentities.getJSONObject(i));
identity.account = account.id;
db.identity().insertIdentity(identity);
}
}
} catch (Throwable ex) {
Log.e(ex);
}
}
public static synchronized DB getInstance(Context context) {
int apid = android.os.Process.myPid();
Context acontext = context.getApplicationContext();
if (sInstance != null &&
(sPid != apid || !Objects.equals(sContext, acontext)))
try {
Log.e("Orphan database instance pid=" + apid + "/" + sPid);
sInstance = null;
} catch (Throwable ex) {
Log.e(ex);
}
sPid = apid;
sContext = acontext;
if (sInstance == null) {
Log.i("Creating database instance pid=" + sPid);
sInstance = migrate(sContext, getBuilder(sContext)).build();
Helper.getSerialExecutor().execute(new Runnable() {
@Override
public void run() {
checkEmergencyBackup(sContext);
}
});
try {
Log.i("Disabling view invalidation");
Field fmViewTables = InvalidationTracker.class.getDeclaredField("mViewTables");
fmViewTables.setAccessible(true);
Map> mViewTables = (Map) fmViewTables.get(sInstance.getInvalidationTracker());
mViewTables.get("account_view").clear();
mViewTables.get("identity_view").clear();
mViewTables.get("folder_view").clear();
Log.i("Disabled view invalidation");
} catch (ReflectiveOperationException ex) {
Log.e(ex);
}
sInstance.getInvalidationTracker().addObserver(new InvalidationTracker.Observer(DB_TABLES) {
@Override
public void onInvalidated(@NonNull Set tables) {
Log.d("ROOM invalidated=" + TextUtils.join(",", tables));
}
});
}
return sInstance;
}
private static RoomDatabase.Builder getBuilder(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean wal = prefs.getBoolean("wal", true);
Log.i("DB wal=" + wal);
RoomDatabase.Builder builder = Room
.databaseBuilder(context, DB.class, DB_NAME)
//.openHelperFactory(new RequerySQLiteOpenHelperFactory())
//.setQueryExecutor()
.setTransactionExecutor(executor)
.setJournalMode(wal ? JournalMode.WRITE_AHEAD_LOGGING : JournalMode.TRUNCATE) // using the latest sqlite
.addCallback(new Callback() {
@Override
public void onCreate(@NonNull SupportSQLiteDatabase db) {
defaultSearches(db, context);
}
@Override
public void onOpen(@NonNull SupportSQLiteDatabase db) {
try {
Map crumb = new HashMap<>();
crumb.put("version", Integer.toString(db.getVersion()));
crumb.put("WAL", Boolean.toString(db.isWriteAheadLoggingEnabled()));
Log.breadcrumb("Database", crumb);
// https://www.sqlite.org/pragma.html#pragma_auto_vacuum
// https://android.googlesource.com/platform/external/sqlite.git/+/6ab557bdc070f11db30ede0696888efd19800475%5E!/
boolean sqlite_auto_vacuum = prefs.getBoolean("sqlite_auto_vacuum", false);
String mode = (sqlite_auto_vacuum ? "FULL" : "INCREMENTAL");
Log.i("Set PRAGMA auto_vacuum=" + mode);
try (Cursor cursor = db.query("PRAGMA auto_vacuum=" + mode + ";")) {
cursor.moveToNext(); // required
}
// https://sqlite.org/pragma.html#pragma_synchronous
boolean sqlite_sync_extra = prefs.getBoolean("sqlite_sync_extra", true);
String sync = (sqlite_sync_extra ? "EXTRA" : "NORMAL");
Log.i("Set PRAGMA synchronous=" + sync);
try (Cursor cursor = db.query("PRAGMA synchronous=" + sync + ";")) {
cursor.moveToNext(); // required
}
// https://www.sqlite.org/pragma.html#pragma_journal_size_limit
Log.i("Set PRAGMA journal_size_limit=" + DB_JOURNAL_SIZE_LIMIT);
try (Cursor cursor = db.query("PRAGMA journal_size_limit=" + DB_JOURNAL_SIZE_LIMIT + ";")) {
cursor.moveToNext(); // required
}
// https://www.sqlite.org/pragma.html#pragma_cache_size
Integer cache_size = getCacheSizeKb(context);
if (cache_size != null) {
cache_size = -cache_size; // kibibytes
Log.i("Set PRAGMA cache_size=" + cache_size);
// TODO CASA PRAGMA does not support placeholders
try (Cursor cursor = db.query("PRAGMA cache_size=" + cache_size + ";")) {
cursor.moveToNext(); // required
}
}
// Prevent long running operations from getting an exclusive lock
// https://www.sqlite.org/pragma.html#pragma_cache_spill
Log.i("Set PRAGMA cache_spill=0");
try (Cursor cursor = db.query("PRAGMA cache_spill=0;")) {
cursor.moveToNext(); // required
}
Log.i("Set PRAGMA recursive_triggers=off");
try (Cursor cursor = db.query("PRAGMA recursive_triggers=off;")) {
cursor.moveToNext(); // required
}
// https://www.sqlite.org/pragma.html
for (String pragma : DB_PRAGMAS)
if (!"compile_options".equals(pragma) || BuildConfig.DEBUG) {
// TODO CASA PRAGMA does not support placeholders
try (Cursor cursor = db.query("PRAGMA " + pragma + ";")) {
boolean has = false;
while (cursor.moveToNext()) {
has = true;
Log.i("Get PRAGMA " + pragma + "=" + (cursor.isNull(0) ? "" : cursor.getString(0)));
}
if (!has)
Log.i("Get PRAGMA " + pragma + "=>");
} catch (Throwable ex) {
Log.e(ex);
}
}
if (BuildConfig.DEBUG && false)
dropTriggers(db);
createTriggers(db);
ContentValues cv = new ContentValues();
cv.put("host", "imap.mnet-online.de");
int rows = db.update(
"account",
SQLiteDatabase.CONFLICT_ABORT,
cv,
"host = ? AND (port = ? OR port = ?)",
new Object[]{"mail.m-online.net", 143, 993});
if (rows > 0)
EntityLog.log(context, "M-net updated");
} catch (Throwable ex) {
/*
at eu.faircode.email.DB$6.onOpen(DB.java:522)
at eu.faircode.email.DB_Impl$1.onOpen(DB_Impl.java:171)
at androidx.room.RoomOpenHelper.onOpen(RoomOpenHelper.java:136)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.onOpen(FrameworkSQLiteOpenHelper.kt:287)
at android.database.sqlite.SQLiteOpenHelper.getDatabaseLocked(SQLiteOpenHelper.java:427)
at android.database.sqlite.SQLiteOpenHelper.getWritableDatabase(SQLiteOpenHelper.java:316)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.getWritableOrReadableDatabase(FrameworkSQLiteOpenHelper.kt:232)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.innerGetDatabase(FrameworkSQLiteOpenHelper.kt:190)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper$OpenHelper.getSupportDatabase(FrameworkSQLiteOpenHelper.kt:151)
at androidx.sqlite.db.framework.FrameworkSQLiteOpenHelper.getWritableDatabase(FrameworkSQLiteOpenHelper.kt:104)
at androidx.room.RoomDatabase.inTransaction(RoomDatabase.java:706)
*/
Log.forceCrashReporting();
Log.e(ex);
// FrameworkSQLiteOpenHelper.innerGetDatabase will delete the database
throw ex;
}
}
@Override
public void onDestructiveMigration(@NonNull SupportSQLiteDatabase db) {
Log.e("WTF destructive migration");
}
});
if (BuildConfig.DEBUG && false)
builder.setQueryCallback(new QueryCallback() {
@Override
public void onQuery(@NonNull String sqlQuery, @NonNull List