mirror
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Cloud sync: added worker

This commit is contained in:
M66B 2023-01-16 15:09:53 +01:00
parent 91867ce93b
commit 02e06b5253
4 changed files with 322 additions and 195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/src/main/java/eu/faircode/email/ApplicationEx.java
View File

@ -258,6 +258,7 @@ public class ApplicationEx extends Application
WorkerAutoUpdate.init(this);
WorkerCleanup.init(this);
WorkerDailyRules.init(this);
WorkerSync.init(this);
}
registerReceiver(onScreenOff, new IntentFilter(Intent.ACTION_SCREEN_OFF));

208
app/src/main/java/eu/faircode/email/CloudSync.java
View File

@ -20,9 +20,13 @@ package eu.faircode.email;
*/
import android.content.Context;
import android.content.SharedPreferences;
import android.text.TextUtils;
import android.util.Base64;
import android.util.Pair;
import androidx.preference.PreferenceManager;
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
@ -37,6 +41,7 @@ import java.security.spec.InvalidKeySpecException;
import java.security.spec.KeySpec;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -55,13 +60,210 @@ public class CloudSync {
private static final Map<String, Pair<byte[], byte[]>> keyCache = new HashMap<>();
public static JSONObject perform(Context context, String user, String password, String command, JSONObject jrequest)
// Upper level
static void execute(Context context, String command)
throws JSONException, GeneralSecurityException, IOException {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String user = prefs.getString("cloud_user", null);
String password = prefs.getString("cloud_password", null);
JSONObject jrequest = new JSONObject();
if ("sync".equals(command)) {
DB db = DB.getInstance(context);
long lrevision = prefs.getLong("sync_status", new Date().getTime());
Log.i("Cloud sync status=" + lrevision);
for (EntitySync s : db.sync().getSync(null, null, Long.MAX_VALUE))
Log.i("Cloud sync " + s.entity + ":" + s.reference + " " + s.action + " " + new Date(s.time));
db.sync().deleteSync(Long.MAX_VALUE);
JSONObject jsyncstatus = new JSONObject();
jsyncstatus.put("key", "sync.status");
jsyncstatus.put("rev", lrevision);
JSONArray jitems = new JSONArray();
jitems.put(jsyncstatus);
jrequest.put("items", jitems);
JSONObject jresponse = call(context, user, password, "read", jrequest);
jitems = jresponse.getJSONArray("items");
if (jitems.length() == 0) {
Log.i("Cloud server is empty");
sendLocalData(context, user, password, lrevision);
} else if (jitems.length() == 1) {
Log.i("Cloud sync check");
jsyncstatus = jitems.getJSONObject(0);
receiveRemoteData(context, user, password, lrevision, jsyncstatus);
} else
throw new IllegalArgumentException("Expected one status item");
} else {
JSONArray jitems = new JSONArray();
jrequest.put("items", jitems);
call(context, user, password, command, jrequest);
}
}
private static void sendLocalData(Context context, String user, String password, long lrevision) throws JSONException, GeneralSecurityException, IOException {
DB db = DB.getInstance(context);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
List<EntityAccount> accounts = db.account().getSynchronizingAccounts(null);
Log.i("Cloud accounts=" + (accounts == null ? null : accounts.size()));
if (accounts == null || accounts.size() == 0)
return;
JSONArray jupload = new JSONArray();
JSONArray jaccountuuids = new JSONArray();
for (EntityAccount account : accounts)
if (!TextUtils.isEmpty(account.uuid)) {
jaccountuuids.put(account.uuid);
JSONArray jidentitieuuids = new JSONArray();
List<EntityIdentity> identities = db.identity().getIdentities(account.id);
if (identities != null)
for (EntityIdentity identity : identities)
if (!TextUtils.isEmpty(identity.uuid)) {
jidentitieuuids.put(identity.uuid);
JSONObject jidentity = new JSONObject();
jidentity.put("key", "identity." + identity.uuid);
jidentity.put("val", identity.toJSON().toString());
jidentity.put("rev", lrevision);
jupload.put(jidentity);
}
JSONObject jaccountdata = new JSONObject();
jaccountdata.put("account", account.toJSON());
jaccountdata.put("identities", jidentitieuuids);
JSONObject jaccount = new JSONObject();
jaccount.put("key", "account." + account.uuid);
jaccount.put("val", jaccountdata.toString());
jaccount.put("rev", lrevision);
jupload.put(jaccount);
}
JSONObject jaccountuuidsholder = new JSONObject();
jaccountuuidsholder.put("uuids", jaccountuuids);
JSONObject jaccountstatus = new JSONObject();
jaccountstatus.put("accounts", jaccountuuidsholder);
JSONObject jsyncstatus = new JSONObject();
jsyncstatus.put("key", "sync.status");
jsyncstatus.put("val", jaccountstatus.toString());
jsyncstatus.put("rev", lrevision);
jupload.put(jsyncstatus);
JSONObject jrequest = new JSONObject();
jrequest.put("items", jupload);
call(context, user, password, "write", jrequest);
prefs.edit().putLong("sync_status", lrevision).apply();
}
private static void receiveRemoteData(Context context, String user, String password, long lrevision, JSONObject jsyncstatus) throws JSONException, GeneralSecurityException, IOException {
DB db = DB.getInstance(context);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
long rrevision = jsyncstatus.getLong("rev");
Log.i("Cloud revision=" + lrevision + "/" + rrevision);
if (BuildConfig.DEBUG)
lrevision--;
if (rrevision <= lrevision)
return; // no changes
// New revision
JSONArray jdownload = new JSONArray();
// Get accounts
JSONObject jstatus = new JSONObject(jsyncstatus.getString("val"));
JSONObject jaccountstatus = jstatus.getJSONObject("accounts");
JSONArray jaccountuuids = jaccountstatus.getJSONArray("uuids");
for (int i = 0; i < jaccountuuids.length(); i++) {
String uuid = jaccountuuids.getString(i);
JSONObject jaccount = new JSONObject();
jaccount.put("key", "account." + uuid);
jaccount.put("rev", lrevision);
jdownload.put(jaccount);
Log.i("Cloud account " + uuid);
}
if (jdownload.length() > 0) {
Log.i("Cloud getting accounts");
JSONObject jrequest = new JSONObject();
jrequest.put("items", jdownload);
JSONObject jresponse = call(context, user, password, "sync", jrequest);
// Process accounts
Log.i("Cloud processing accounts");
JSONArray jitems = jresponse.getJSONArray("items");
jdownload = new JSONArray();
for (int i = 0; i < jitems.length(); i++) {
JSONObject jaccount = jitems.getJSONObject(i);
String value = jaccount.getString("val");
long revision = jaccount.getLong("rev");
JSONObject jaccountdata = new JSONObject(value);
EntityAccount raccount = EntityAccount.fromJSON(jaccountdata.getJSONObject("account"));
EntityAccount laccount = db.account().getAccountByUUID(raccount.uuid);
JSONArray jidentities = jaccountdata.getJSONArray("identities");
Log.i("Cloud account " + raccount.uuid + "=" + (laccount == null ? "insert" : "update") +
" rev=" + revision +
" identities=" + jidentities +
" size=" + value.length());
for (int j = 0; j < jidentities.length(); j++) {
JSONObject jidentity = new JSONObject();
jidentity.put("key", "identity." + jidentities.getString(j));
jidentity.put("rev", lrevision);
jdownload.put(jidentity);
}
}
if (jdownload.length() > 0) {
// Get identities
Log.i("Cloud getting identities");
jrequest.put("items", jdownload);
jresponse = call(context, user, password, "sync", jrequest);
// Process identities
Log.i("Cloud processing identities");
jitems = jresponse.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++) {
JSONObject jidentity = jitems.getJSONObject(i);
String value = jidentity.getString("val");
long revision = jidentity.getLong("rev");
EntityIdentity ridentity = EntityIdentity.fromJSON(new JSONObject(value));
EntityIdentity lidentity = db.identity().getIdentityByUUID(ridentity.uuid);
Log.i("Cloud identity " + ridentity.uuid + "=" + (lidentity == null ? "insert" : "update") +
" rev=" + revision +
" size=" + value.length());
}
}
}
prefs.edit().putLong("sync_status", rrevision).apply();
}
// Lower level
public static JSONObject call(Context context, String user, String password, String command, JSONObject jrequest)
throws GeneralSecurityException, JSONException, IOException {
Log.i("Cloud command=" + command);
jrequest.put("command", command);
List<JSONObject> responses = new ArrayList<>();
for (JSONArray batch : partition(jrequest.getJSONArray("items"))) {
jrequest.put("items", batch);
responses.add(_perform(context, user, password, jrequest));
responses.add(_call(context, user, password, jrequest));
}
if (responses.size() == 1)
return responses.get(0);
@ -78,7 +280,7 @@ public class CloudSync {
}
}
private static JSONObject _perform(Context context, String user, String password, JSONObject jrequest)
private static JSONObject _call(Context context, String user, String password, JSONObject jrequest)
throws GeneralSecurityException, JSONException, IOException {
byte[] salt = MessageDigest.getInstance("SHA256").digest(user.getBytes());
byte[] huser = MessageDigest.getInstance("SHA256").digest(salt);

201
app/src/main/java/eu/faircode/email/FragmentOptionsBackup.java
View File

@ -83,7 +83,6 @@ import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
import java.security.spec.KeySpec;
import java.text.DateFormat;
@ -1515,9 +1514,6 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
private void cloud(Bundle args) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(getContext());
args.putString("user", prefs.getString("cloud_user", null));
args.putString("password", prefs.getString("cloud_password", null));
new SimpleTask<Void>() {
@Override
protected void onPreExecute(Bundle args) {
@ -1531,46 +1527,8 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
@Override
protected Void onExecute(Context context, Bundle args) throws Throwable {
String user = args.getString("user");
String password = args.getString("password");
String command = args.getString("command");
JSONObject jrequest = new JSONObject();
if ("sync".equals(command)) {
DB db = DB.getInstance(context);
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
long lrevision = prefs.getLong("sync_status", new Date().getTime());
Log.i("Cloud sync status=" + lrevision);
for (EntitySync s : db.sync().getSync(null, null, Long.MAX_VALUE))
Log.i("Cloud sync " + s.entity + ":" + s.reference + " " + s.action + " " + new Date(s.time));
db.sync().deleteSync(Long.MAX_VALUE);
JSONObject jsyncstatus = new JSONObject();
jsyncstatus.put("key", "sync.status");
jsyncstatus.put("rev", lrevision);
JSONArray jitems = new JSONArray();
jitems.put(jsyncstatus);
jrequest.put("items", jitems);
JSONObject jresponse = CloudSync.perform(context, user, password, "read", jrequest);
jitems = jresponse.getJSONArray("items");
if (jitems.length() == 0) {
Log.i("Cloud server is empty");
sendLocalData(context, user, password, lrevision);
} else if (jitems.length() == 1) {
Log.i("Cloud sync check");
jsyncstatus = jitems.getJSONObject(0);
receiveRemoteData(context, user, password, lrevision, jsyncstatus);
} else
throw new IllegalArgumentException("Expected one status item");
} else
CloudSync.perform(context, user, password, command, jrequest);
CloudSync.execute(context, command);
return null;
}
@ -1582,11 +1540,9 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
.remove("cloud_user")
.remove("cloud_password")
.apply();
else
prefs.edit()
.putString("cloud_user", args.getString("user"))
.putString("cloud_password", args.getString("password"))
.apply();
if ("login".equals(command) || "logout".equals(command))
WorkerSync.init(getContext());
view.post(new Runnable() {
@Override
@ -1599,6 +1555,11 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
@Override
protected void onException(Bundle args, Throwable ex) {
if (ex instanceof SecurityException) {
prefs.edit()
.remove("cloud_user")
.remove("cloud_password")
.apply();
AlertDialog.Builder builder = new AlertDialog.Builder(getContext())
.setIcon(R.drawable.twotone_warning_24)
.setTitle(getString(R.string.title_advanced_cloud_invalid))
@ -1610,150 +1571,6 @@ public class FragmentOptionsBackup extends FragmentBase implements SharedPrefere
} else
Log.unexpectedError(getParentFragmentManager(), ex);
}
private void sendLocalData(Context context, String user, String password, long lrevision) throws JSONException, GeneralSecurityException, IOException {
DB db = DB.getInstance(context);
List<EntityAccount> accounts = db.account().getSynchronizingAccounts(null);
Log.i("Cloud accounts=" + (accounts == null ? null : accounts.size()));
if (accounts == null || accounts.size() == 0)
return;
JSONArray jupload = new JSONArray();
JSONArray jaccountuuids = new JSONArray();
for (EntityAccount account : accounts)
if (!TextUtils.isEmpty(account.uuid)) {
jaccountuuids.put(account.uuid);
JSONArray jidentitieuuids = new JSONArray();
List<EntityIdentity> identities = db.identity().getIdentities(account.id);
if (identities != null)
for (EntityIdentity identity : identities)
if (!TextUtils.isEmpty(identity.uuid)) {
jidentitieuuids.put(identity.uuid);
JSONObject jidentity = new JSONObject();
jidentity.put("key", "identity." + identity.uuid);
jidentity.put("val", identity.toJSON().toString());
jidentity.put("rev", lrevision);
jupload.put(jidentity);
}
JSONObject jaccountdata = new JSONObject();
jaccountdata.put("account", account.toJSON());
jaccountdata.put("identities", jidentitieuuids);
JSONObject jaccount = new JSONObject();
jaccount.put("key", "account." + account.uuid);
jaccount.put("val", jaccountdata.toString());
jaccount.put("rev", lrevision);
jupload.put(jaccount);
}
JSONObject jaccountuuidsholder = new JSONObject();
jaccountuuidsholder.put("uuids", jaccountuuids);
JSONObject jaccountstatus = new JSONObject();
jaccountstatus.put("accounts", jaccountuuidsholder);
JSONObject jsyncstatus = new JSONObject();
jsyncstatus.put("key", "sync.status");
jsyncstatus.put("val", jaccountstatus.toString());
jsyncstatus.put("rev", lrevision);
jupload.put(jsyncstatus);
JSONObject jrequest = new JSONObject();
jrequest.put("items", jupload);
CloudSync.perform(context, user, password, "write", jrequest);
prefs.edit().putLong("sync_status", lrevision).apply();
}
private void receiveRemoteData(Context context, String user, String password, long lrevision, JSONObject jsyncstatus) throws JSONException, GeneralSecurityException, IOException {
DB db = DB.getInstance(context);
long rrevision = jsyncstatus.getLong("rev");
Log.i("Cloud revision=" + lrevision + "/" + rrevision);
if (BuildConfig.DEBUG)
lrevision--;
if (rrevision <= lrevision)
return; // no changes
// New revision
JSONArray jdownload = new JSONArray();
// Get accounts
JSONObject jstatus = new JSONObject(jsyncstatus.getString("val"));
JSONObject jaccountstatus = jstatus.getJSONObject("accounts");
JSONArray jaccountuuids = jaccountstatus.getJSONArray("uuids");
for (int i = 0; i < jaccountuuids.length(); i++) {
String uuid = jaccountuuids.getString(i);
JSONObject jaccount = new JSONObject();
jaccount.put("key", "account." + uuid);
jaccount.put("rev", lrevision);
jdownload.put(jaccount);
Log.i("Cloud account " + uuid);
}
if (jdownload.length() > 0) {
Log.i("Cloud getting accounts");
JSONObject jrequest = new JSONObject();
jrequest.put("items", jdownload);
JSONObject jresponse = CloudSync.perform(context, user, password, "sync", jrequest);
// Process accounts
Log.i("Cloud processing accounts");
JSONArray jitems = jresponse.getJSONArray("items");
jdownload = new JSONArray();
for (int i = 0; i < jitems.length(); i++) {
JSONObject jaccount = jitems.getJSONObject(i);
String value = jaccount.getString("val");
long revision = jaccount.getLong("rev");
JSONObject jaccountdata = new JSONObject(value);
EntityAccount raccount = EntityAccount.fromJSON(jaccountdata.getJSONObject("account"));
EntityAccount laccount = db.account().getAccountByUUID(raccount.uuid);
JSONArray jidentities = jaccountdata.getJSONArray("identities");
Log.i("Cloud account " + raccount.uuid + "=" + (laccount == null ? "insert" : "update") +
" rev=" + revision +
" identities=" + jidentities +
" size=" + value.length());
for (int j = 0; j < jidentities.length(); j++) {
JSONObject jidentity = new JSONObject();
jidentity.put("key", "identity." + jidentities.getString(j));
jidentity.put("rev", lrevision);
jdownload.put(jidentity);
}
}
if (jdownload.length() > 0) {
// Get identities
Log.i("Cloud getting identities");
jrequest.put("items", jdownload);
jresponse = CloudSync.perform(context, user, password, "sync", jrequest);
// Process identities
Log.i("Cloud processing identities");
jitems = jresponse.getJSONArray("items");
for (int i = 0; i < jitems.length(); i++) {
JSONObject jidentity = jitems.getJSONObject(i);
String value = jidentity.getString("val");
long revision = jidentity.getLong("rev");
EntityIdentity ridentity = EntityIdentity.fromJSON(new JSONObject(value));
EntityIdentity lidentity = db.identity().getIdentityByUUID(ridentity.uuid);
Log.i("Cloud identity " + ridentity.uuid + "=" + (lidentity == null ? "insert" : "update") +
" rev=" + revision +
" size=" + value.length());
}
}
}
prefs.edit().putLong("sync_status", rrevision).apply();
}
}.execute(FragmentOptionsBackup.this, args, "cloud");
}

107
app/src/main/java/eu/faircode/email/WorkerSync.java Normal file
View File

@ -0,0 +1,107 @@
package eu.faircode.email;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see <http://www.gnu.org/licenses/>.
Copyright 2018-2023 by Marcel Bokhorst (M66B)
*/
import static android.os.Process.THREAD_PRIORITY_BACKGROUND;
import android.content.Context;
import android.content.SharedPreferences;
import android.text.TextUtils;
import androidx.annotation.NonNull;
import androidx.preference.PreferenceManager;
import androidx.work.ExistingPeriodicWorkPolicy;
import androidx.work.PeriodicWorkRequest;
import androidx.work.WorkManager;
import androidx.work.Worker;
import androidx.work.WorkerParameters;
import java.util.Calendar;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
public class WorkerSync extends Worker {
private static final Semaphore semaphore = new Semaphore(1);
public WorkerSync(@NonNull Context context, @NonNull WorkerParameters workerParams) {
super(context, workerParams);
Log.i("Instance " + getName());
}
@NonNull
@Override
public Result doWork() {
Thread.currentThread().setPriority(THREAD_PRIORITY_BACKGROUND);
final Context context = getApplicationContext();
try {
semaphore.acquire();
EntityLog.log(context, EntityLog.Type.Rules, "Cloud sync execute");
CloudSync.execute(context, "sync");
EntityLog.log(context, EntityLog.Type.Rules, "Cloud sync completed");
return Result.success();
} catch (Throwable ex) {
Log.e(ex);
return Result.failure();
} finally {
semaphore.release();
}
}
static void init(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String user = prefs.getString("cloud_user", null);
String password = prefs.getString("cloud_password", null);
boolean enabled = false && !(TextUtils.isEmpty(user) || TextUtils.isEmpty(password));
try {
if (enabled) {
Calendar cal = Calendar.getInstance();
long delay = cal.getTimeInMillis();
cal.set(Calendar.MILLISECOND, 0);
cal.set(Calendar.SECOND, 0);
cal.set(Calendar.MINUTE, 0);
cal.set(Calendar.HOUR_OF_DAY, 1);
cal.add(Calendar.DAY_OF_MONTH, 1);
delay = cal.getTimeInMillis() - delay;
Log.i("Queuing " + getName() + " delay=" + (delay / (60 * 1000L)) + "m");
PeriodicWorkRequest.Builder builder =
new PeriodicWorkRequest.Builder(WorkerSync.class, 1, TimeUnit.DAYS)
.setInitialDelay(delay, TimeUnit.MILLISECONDS);
WorkManager.getInstance(context)
.enqueueUniquePeriodicWork(getName(), ExistingPeriodicWorkPolicy.KEEP, builder.build());
Log.i("Queued " + getName());
} else {
Log.i("Cancelling " + getName());
WorkManager.getInstance(context).cancelUniqueWork(getName());
Log.i("Cancelled " + getName());
}
} catch (IllegalStateException ex) {
// https://issuetracker.google.com/issues/138465476
Log.w(ex);
}
}
private static String getName() {
return WorkerSync.class.getSimpleName();
}
}