From a985b1cec9e9250e985d2c82a247ccc17ff021f6 Mon Sep 17 00:00:00 2001 From: M66B Date: Tue, 19 Jan 2021 09:04:48 +0100 Subject: [PATCH] Enable cld3 --- ATTRIBUTION.md | 1 + README.md | 2 +- app/src/main/assets/ATTRIBUTION.md | 1 + .../eu/faircode/email/FragmentMessages.java | 12 +++--- .../email/FragmentOptionsBehavior.java | 1 - .../java/eu/faircode/email/HtmlHelper.java | 2 +- .../java/eu/faircode/email/TextHelper.java | 43 ++++++++++--------- .../eu/faircode/email/ViewModelMessages.java | 7 +-- app/src/main/res/values/strings.xml | 2 +- metadata/en-US/full_description.txt | 2 +- 10 files changed, 37 insertions(+), 36 deletions(-) diff --git a/ATTRIBUTION.md b/ATTRIBUTION.md index f60a3a7c0f..beded60b13 100644 --- a/ATTRIBUTION.md +++ b/ATTRIBUTION.md @@ -28,4 +28,5 @@ FairEmail uses: * [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE). * [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE) * [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE). +* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE). * [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing). diff --git a/README.md b/README.md index 85e8466e10..ed08a98aeb 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ FairEmail might be for you if you value your privacy. * Offline storage and operations * Battery friendly * Low data usage -* Small (~ 15 MB) +* Small (< 20 MB) * Material design (including dark/black theme) * Maintained and supported diff --git a/app/src/main/assets/ATTRIBUTION.md b/app/src/main/assets/ATTRIBUTION.md index f60a3a7c0f..beded60b13 100644 --- a/app/src/main/assets/ATTRIBUTION.md +++ b/app/src/main/assets/ATTRIBUTION.md @@ -28,4 +28,5 @@ FairEmail uses: * [Disconnect's tracker protection lists](https://github.com/disconnectme/disconnect-tracking-protection). Copyright 2010-2020 Disconnect, Inc. [Creative Commons Attribution-NonCommercial-ShareAlike 4.0 International license](https://github.com/disconnectme/disconnect-tracking-protection/blob/master/LICENSE). * [Over-Scroll Support For Android's RecyclerView, ListView, GridView, ScrollView ...](https://github.com/EverythingMe/overscroll-decor). Copyright (c) 2015, DoAT Media Ltd. [BSD-2-Clause License](https://github.com/EverythingMe/overscroll-decor/blob/master/LICENSE) * [Compact Encoding Detection](https://github.com/google/compact_enc_det). Copyright 2016 Google Inc. [Apache License 2.0](https://github.com/google/compact_enc_det/blob/master/LICENSE). +* [Compact Language Detector v3](https://github.com/google/cld3). Copyright 2016 Google Inc. All rights reserved. [Apache License 2.0](https://github.com/google/cld3/blob/master/LICENSE). * [POI-HMEF](https://poi.apache.org/components/hmef/index.html). Copyright © 2001-2020 The Apache Software Foundation. [Apache Software License v2](https://poi.apache.org/devel/guidelines.html#The+Licensing). diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java index b6c564235a..3ee8f57895 100644 --- a/app/src/main/java/eu/faircode/email/FragmentMessages.java +++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java @@ -3983,8 +3983,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. (viewType == AdapterMessage.ViewType.UNIFIED || (viewType == AdapterMessage.ViewType.FOLDER && !outbox)); - String filter_language = prefs.getString("filter_language", null); - boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language)); + boolean filter_active = (filter_seen || filter_unflagged || filter_unknown); MenuItem menuFilter = menu.findItem(R.id.menu_filter); menuFilter.setShowAsAction(folder && filter_active ? MenuItem.SHOW_AS_ACTION_ALWAYS @@ -4055,8 +4054,7 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. menu.findItem(R.id.menu_compact).setChecked(compact); - menu.findItem(R.id.menu_select_language).setVisible( - language_detection && folder && Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q); + menu.findItem(R.id.menu_select_language).setVisible(language_detection && folder); menu.findItem(R.id.menu_select_all).setVisible(folder); menu.findItem(R.id.menu_select_found).setVisible(viewType == AdapterMessage.ViewType.SEARCH); menu.findItem(R.id.menu_mark_all_read).setVisible(folder); @@ -4421,8 +4419,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false); boolean filter_unknown = prefs.getBoolean("filter_unknown", false); boolean filter_snoozed = prefs.getBoolean("filter_snoozed", true); - String filter_language = prefs.getString("filter_language", null); boolean language_detection = prefs.getBoolean("language_detection", false); + String filter_language = prefs.getString("filter_language", null); DB db = DB.getInstance(context); try { @@ -4683,8 +4681,10 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. boolean filter_seen = prefs.getBoolean("filter_seen", false); boolean filter_unflagged = prefs.getBoolean("filter_unflagged", false); boolean filter_unknown = prefs.getBoolean("filter_unknown", false); + boolean language_detection = prefs.getBoolean("language_detection", false); String filter_language = prefs.getString("filter_language", null); - boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || !TextUtils.isEmpty(filter_language)); + boolean filter_active = (filter_seen || filter_unflagged || filter_unknown || + (language_detection && !TextUtils.isEmpty(filter_language))); boolean none = (items == 0 && !loading && tasks == 0 && initialized); boolean filtered = (filter_active && viewType != AdapterMessage.ViewType.SEARCH); diff --git a/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java b/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java index e29d5353ed..f0d28aefaa 100644 --- a/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java +++ b/app/src/main/java/eu/faircode/email/FragmentOptionsBehavior.java @@ -170,7 +170,6 @@ public class FragmentOptionsBehavior extends FragmentBase implements SharedPrefe } }); - swLanguageDetection.setEnabled(Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q); swLanguageDetection.setOnCheckedChangeListener(new CompoundButton.OnCheckedChangeListener() { @Override public void onCheckedChanged(CompoundButton compoundButton, boolean checked) { diff --git a/app/src/main/java/eu/faircode/email/HtmlHelper.java b/app/src/main/java/eu/faircode/email/HtmlHelper.java index 925392abc3..5b5f925542 100644 --- a/app/src/main/java/eu/faircode/email/HtmlHelper.java +++ b/app/src/main/java/eu/faircode/email/HtmlHelper.java @@ -1716,7 +1716,7 @@ public class HtmlHelper { if (!language_detection) return null; - String text = getPreview(body); + String text = getFullText(body); Locale locale = TextHelper.detectLanguage(context, text); return (locale == null ? null : locale.getLanguage()); } catch (Throwable ex) { diff --git a/app/src/main/java/eu/faircode/email/TextHelper.java b/app/src/main/java/eu/faircode/email/TextHelper.java index 8a28ce27c2..e135187be1 100644 --- a/app/src/main/java/eu/faircode/email/TextHelper.java +++ b/app/src/main/java/eu/faircode/email/TextHelper.java @@ -27,7 +27,6 @@ import android.view.textclassifier.ConversationAction; import android.view.textclassifier.ConversationActions; import android.view.textclassifier.TextClassificationManager; import android.view.textclassifier.TextClassifier; -import android.view.textclassifier.TextLanguage; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -41,6 +40,8 @@ import java.util.Locale; import java.util.Set; public class TextHelper { + private static final int MAX_SAMPLE_SIZE = 8192; + static { System.loadLibrary("fairemail"); } @@ -48,33 +49,35 @@ public class TextHelper { private static native DetectResult jni_detect_language(byte[] octets); static Locale detectLanguage(Context context, String text) { - // Why not ML kit? https://developers.google.com/ml-kit/terms + // Why not ML kit? + // https://developers.google.com/ml-kit/terms + if (TextUtils.isEmpty(text)) return null; - if (BuildConfig.DEBUG) { - // https://github.com/google/cld3 - DetectResult result = jni_detect_language(text.getBytes()); - Log.i("Language=" + result); - return Locale.forLanguageTag(result.language); + byte[] octets = text.getBytes(); + byte[] sample; + if (octets.length < MAX_SAMPLE_SIZE) + sample = octets; + else { + sample = new byte[MAX_SAMPLE_SIZE]; + System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE); } - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) + // https://github.com/google/cld3 + Log.i("cld3 sample=" + sample.length); + DetectResult result = jni_detect_language(sample); + Log.i("Language=" + result); + + if (result.probability < 0.5) return null; - // https://issuetracker.google.com/issues/173337263 - TextClassificationManager tcm = - (TextClassificationManager) context.getSystemService(Context.TEXT_CLASSIFICATION_SERVICE); - if (tcm == null) + try { + return Locale.forLanguageTag(result.language); + } catch (Throwable ex) { + Log.w(ex); return null; - - TextLanguage.Request request = new TextLanguage.Request.Builder(text).build(); - TextClassifier tc = tcm.getTextClassifier(); - TextLanguage tlanguage = tc.detectLanguage(request); - if (tlanguage.getLocaleHypothesisCount() > 0) - return tlanguage.getLocale(0).toLocale(); - - return null; + } } static ConversationActions getConversationActions( diff --git a/app/src/main/java/eu/faircode/email/ViewModelMessages.java b/app/src/main/java/eu/faircode/email/ViewModelMessages.java index 5a6f190e34..f1d6b720c3 100644 --- a/app/src/main/java/eu/faircode/email/ViewModelMessages.java +++ b/app/src/main/java/eu/faircode/email/ViewModelMessages.java @@ -21,7 +21,6 @@ package eu.faircode.email; import android.content.Context; import android.content.SharedPreferences; -import android.os.Build; import android.os.Bundle; import android.text.TextUtils; import android.util.Pair; @@ -449,10 +448,8 @@ public class ViewModelMessages extends ViewModel { this.filter_snoozed = prefs.getBoolean("filter_snoozed", true); boolean language_detection = prefs.getBoolean("language_detection", false); - if (!language_detection || Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) - this.filter_language = null; - else - this.filter_language = prefs.getString("filter_language", null); + String filter_language = prefs.getString("filter_language", null); + this.filter_language = (language_detection ? filter_language : null); this.debug = prefs.getBoolean("debug", false); } diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 0ef000d83c..1135222748 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -517,7 +517,7 @@ Show frequently used contacts in Android share menu Suggest actions (Android 10+) Suggest reply texts (Android 10+) - Detect message text language (Android 10+) + Detect message text language Build search index %1$d / %2$d messages indexed (%3$s) Classify messages diff --git a/metadata/en-US/full_description.txt b/metadata/en-US/full_description.txt index 4dc718acd1..5fd7d781e5 100644 --- a/metadata/en-US/full_description.txt +++ b/metadata/en-US/full_description.txt @@ -21,7 +21,7 @@ FairEmail might be for you if you value your privacy. * Offline storage and operations * Battery friendly * Low data usage -* Small (~ 15 MB) +* Small (< 20 MB) * Material design (including dark/black theme) * Maintained and supported