FairEmail/app/src/main/java/eu/faircode/email/LanguageTool.java

494 lines
20 KiB
Java
Raw Normal View History

2022-04-27 15:18:08 +00:00
package eu.faircode.email;
/*
This file is part of FairEmail.
FairEmail is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
FairEmail is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with FairEmail. If not, see <http://www.gnu.org/licenses/>.
2023-01-01 07:52:55 +00:00
Copyright 2018-2023 by Marcel Bokhorst (M66B)
2022-04-27 15:18:08 +00:00
*/
import android.content.Context;
2022-07-09 05:47:29 +00:00
import android.content.SharedPreferences;
import android.content.res.Resources;
import android.net.Uri;
2022-10-03 16:31:29 +00:00
import android.os.Build;
import android.os.LocaleList;
2022-10-01 07:51:16 +00:00
import android.text.Editable;
2023-09-13 06:53:24 +00:00
import android.text.SpannableStringBuilder;
2022-10-01 07:51:16 +00:00
import android.text.Spanned;
2022-10-01 09:20:49 +00:00
import android.text.TextUtils;
2022-10-01 07:51:16 +00:00
import android.text.style.SuggestionSpan;
2022-11-13 09:39:05 +00:00
import android.util.Pair;
2022-10-01 07:51:16 +00:00
import android.widget.EditText;
2022-07-09 05:47:29 +00:00
import androidx.core.util.PatternsCompat;
2022-07-09 05:47:29 +00:00
import androidx.preference.PreferenceManager;
2022-04-27 15:18:08 +00:00
import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import java.io.IOException;
import java.io.InputStream;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
2022-04-27 19:04:10 +00:00
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
2022-04-27 15:18:08 +00:00
2023-12-06 14:31:05 +00:00
import javax.net.ssl.HttpsURLConnection;
2022-05-24 17:28:24 +00:00
public class LanguageTool {
2022-10-01 08:45:17 +00:00
static final String LT_URI = "https://api.languagetool.org/v2/";
2022-11-06 11:54:29 +00:00
static final String LT_URI_PLUS = "https://api.languagetoolplus.com/v2/";
2022-11-13 09:39:05 +00:00
2022-04-27 15:18:08 +00:00
private static final int LT_TIMEOUT = 20; // seconds
2022-11-13 09:39:05 +00:00
private static final int LT_MAX_RANGES = 10; // paragraphs
2022-04-27 15:18:08 +00:00
2022-11-08 08:01:19 +00:00
private static JSONArray jlanguages = null;
2022-07-09 05:47:29 +00:00
static boolean isEnabled(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
return prefs.getBoolean("lt_enabled", false);
}
static boolean isAuto(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean lt_enabled = prefs.getBoolean("lt_enabled", false);
2022-10-01 17:34:50 +00:00
boolean lt_auto = prefs.getBoolean("lt_auto", true);
return (lt_enabled && lt_auto);
}
static boolean isSentence(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean lt_enabled = prefs.getBoolean("lt_enabled", false);
boolean lt_sentence = prefs.getBoolean("lt_sentence", false);
return (lt_enabled && lt_sentence);
}
2022-11-08 08:01:19 +00:00
static JSONArray getLanguages(Context context) throws IOException, JSONException {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String lt_uri = prefs.getString("lt_uri", LT_URI_PLUS);
// https://languagetool.org/http-api/swagger-ui/#!/default/get_words
Uri uri = Uri.parse(lt_uri).buildUpon().appendPath("languages").build();
Log.i("LT uri=" + uri);
URL url = new URL(uri.toString());
2023-12-06 14:31:05 +00:00
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
2022-11-08 08:01:19 +00:00
connection.setRequestMethod("GET");
connection.setDoOutput(false);
connection.setReadTimeout(LT_TIMEOUT * 1000);
connection.setConnectTimeout(LT_TIMEOUT * 1000);
ConnectionHelper.setUserAgent(context, connection);
connection.setRequestProperty("Accept", "application/json");
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.connect();
try {
checkStatus(connection);
String response = Helper.readStream(connection.getInputStream());
Log.i("LT response=" + response);
return new JSONArray(response);
} finally {
connection.disconnect();
}
}
2022-04-27 15:18:08 +00:00
static List<Suggestion> getSuggestions(Context context, CharSequence text) throws IOException, JSONException {
if (isPremium(context))
try {
2022-11-13 20:00:35 +00:00
List<Pair<Integer, Integer>> paragraphs = new ArrayList<>();
2022-11-13 20:00:35 +00:00
// Skip links and email addresses
Pattern pattern = Pattern.compile("(" + Helper.EMAIL_ADDRESS + ")" +
"|(" + PatternsCompat.AUTOLINK_WEB_URL.pattern() + ")");
2022-11-13 20:00:35 +00:00
Matcher matcher = pattern.matcher(text);
int index = 0;
while (matcher.find()) {
int start = matcher.start();
int end = matcher.end();
2022-11-13 20:00:35 +00:00
paragraphs.addAll(getParagraphs(index, start, text));
Log.i("LT skipping " + start + "..." + end +
" '" + text.subSequence(start, end).toString().replace('\n', '|') + "'");
index = end;
}
2022-11-13 20:00:35 +00:00
paragraphs.addAll(getParagraphs(index, text.length(), text));
2022-11-13 09:39:05 +00:00
2022-11-13 20:00:35 +00:00
// Get suggestions for paragraphs
for (Pair<Integer, Integer> paragraph : paragraphs)
Log.i("LT paragraph " + paragraph.first + "..." + paragraph.second +
" '" + text.subSequence(paragraph.first, paragraph.second).toString().replace('\n', '|') + "'");
if (paragraphs.size() <= LT_MAX_RANGES) {
List<Suggestion> result = new ArrayList<>();
2022-11-13 20:00:35 +00:00
for (Pair<Integer, Integer> range : paragraphs)
result.addAll(getSuggestions(context, text, range.first, range.second));
return result;
}
} catch (Throwable ex) {
if (BuildConfig.DEBUG)
throw ex;
Log.e(ex);
2022-11-13 09:39:05 +00:00
}
return getSuggestions(context, text, 0, text.length());
}
2022-11-13 20:00:35 +00:00
private static List<Pair<Integer, Integer>> getParagraphs(int from, int to, CharSequence text) {
Log.i("LT paragraphs " + from + "..." + to +
" '" + text.subSequence(from, to).toString().replace('\n', '|') + "'");
2022-11-13 20:00:35 +00:00
List<Pair<Integer, Integer>> paragraphs = new ArrayList<>();
int start = from;
int end = start;
while (end < to) {
while (end < to && text.charAt(end) != '\n')
end++;
if (end > start) {
String fragment = text.subSequence(start, end).toString();
if (!TextUtils.isEmpty(fragment.trim()))
2022-11-13 20:00:35 +00:00
paragraphs.add(new Pair<>(start, end));
}
start = end + 1;
end = start;
}
2022-11-13 20:00:35 +00:00
return paragraphs;
}
private static List<Suggestion> getSuggestions(Context context, CharSequence text, int start, int end) throws IOException, JSONException {
if (start < 0 || end > text.length() || start == end)
2022-10-01 09:20:49 +00:00
return new ArrayList<>();
String t = text.subSequence(start, end).toString();
2022-10-01 09:20:49 +00:00
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
boolean lt_picky = prefs.getBoolean("lt_picky", false);
String lt_user = prefs.getString("lt_user", null);
String lt_key = prefs.getString("lt_key", null);
2022-11-06 11:54:29 +00:00
boolean isPlus = (!TextUtils.isEmpty(lt_user) && !TextUtils.isEmpty(lt_key));
String lt_uri = prefs.getString("lt_uri", isPlus ? LT_URI_PLUS : LT_URI);
2022-04-27 15:18:08 +00:00
// https://languagetool.org/http-api/swagger-ui/#!/default/post_check
Uri.Builder builder = new Uri.Builder()
.appendQueryParameter("text", t)
.appendQueryParameter("language", "auto");
2022-04-28 11:23:28 +00:00
2022-09-04 15:51:33 +00:00
// curl -X GET --header 'Accept: application/json' 'https://api.languagetool.org/v2/languages'
2022-11-08 08:01:19 +00:00
if (jlanguages == null)
jlanguages = getLanguages(context);
2022-09-04 15:51:33 +00:00
List<Locale> locales = new ArrayList<>();
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.N)
locales.add(Locale.getDefault());
else {
LocaleList ll = context.getResources().getConfiguration().getLocales();
for (int i = 0; i < ll.size(); i++)
locales.add(ll.get(i));
2022-04-28 11:23:28 +00:00
}
List<String> code = new ArrayList<>();
for (Locale locale : locales)
for (int i = 0; i < jlanguages.length(); i++) {
JSONObject jlanguage = jlanguages.getJSONObject(i);
String c = jlanguage.optString("longCode");
if (locale.toLanguageTag().equals(c) && c.contains("-")) {
code.add(c);
break;
}
}
if (code.size() > 0)
builder.appendQueryParameter("preferredVariants", TextUtils.join(",", code));
2022-09-15 16:18:59 +00:00
String motherTongue = null;
String slocale = Resources.getSystem().getConfiguration().locale.toLanguageTag();
for (int i = 0; i < jlanguages.length(); i++) {
JSONObject jlanguage = jlanguages.getJSONObject(i);
String c = jlanguage.optString("longCode");
if (TextUtils.isEmpty(c))
continue;
if (slocale.equals(c)) {
motherTongue = c;
break;
}
if (slocale.split("-")[0].equals(c))
motherTongue = c;
}
if (motherTongue != null)
builder.appendQueryParameter("motherTongue", motherTongue);
2022-09-15 16:18:59 +00:00
if (lt_picky)
builder.appendQueryParameter("level", "picky");
2022-11-07 09:51:34 +00:00
if (isPlus)
builder
.appendQueryParameter("username", lt_user)
.appendQueryParameter("apiKey", lt_key);
2022-09-15 16:18:59 +00:00
Uri uri = Uri.parse(lt_uri).buildUpon().appendPath("check").build();
String request = builder.build().toString().substring(1);
2022-04-27 15:18:08 +00:00
Log.i("LT uri=" + uri + " request=" + request);
2022-10-01 08:45:17 +00:00
URL url = new URL(uri.toString());
2023-12-06 14:31:05 +00:00
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
2022-04-27 15:18:08 +00:00
connection.setRequestMethod("POST");
connection.setDoOutput(true);
connection.setReadTimeout(LT_TIMEOUT * 1000);
connection.setConnectTimeout(LT_TIMEOUT * 1000);
ConnectionHelper.setUserAgent(context, connection);
2022-04-27 15:18:08 +00:00
connection.setRequestProperty("Accept", "application/json");
connection.setRequestProperty("Content-Length", Integer.toString(request.length()));
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.connect();
try {
connection.getOutputStream().write(request.getBytes());
2022-11-07 09:51:34 +00:00
checkStatus(connection);
2022-04-27 15:18:08 +00:00
String response = Helper.readStream(connection.getInputStream());
2022-04-27 19:04:10 +00:00
Log.i("LT response=" + response);
2022-04-27 15:18:08 +00:00
List<Suggestion> result = new ArrayList<>();
JSONObject jroot = new JSONObject(response);
JSONArray jmatches = jroot.getJSONArray("matches");
for (int i = 0; i < jmatches.length(); i++) {
JSONObject jmatch = jmatches.getJSONObject(i);
Suggestion suggestion = new Suggestion();
suggestion.title = jmatch.getString("shortMessage");
suggestion.description = jmatch.getString("message");
2022-11-13 09:46:35 +00:00
suggestion.offset = jmatch.getInt("offset") + start;
2022-04-27 15:18:08 +00:00
suggestion.length = jmatch.getInt("length");
JSONArray jreplacements = jmatch.getJSONArray("replacements");
suggestion.replacements = new ArrayList<>();
for (int j = 0; j < jreplacements.length(); j++) {
JSONObject jreplacement = jreplacements.getJSONObject(j);
suggestion.replacements.add(jreplacement.getString("value"));
}
2023-09-11 07:10:19 +00:00
JSONObject jrule = jmatch.optJSONObject("rule");
if (jrule != null)
suggestion.issueType = jrule.optString("issueType");
2023-09-12 20:21:01 +00:00
result.add(suggestion);
2022-04-27 15:18:08 +00:00
}
return result;
} finally {
connection.disconnect();
}
}
2022-11-07 09:51:34 +00:00
static boolean modifyDictionary(Context context, String word, String dictionary, boolean add) throws IOException, JSONException {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String lt_user = prefs.getString("lt_user", null);
String lt_key = prefs.getString("lt_key", null);
String lt_uri = prefs.getString("lt_uri", LT_URI_PLUS);
if (TextUtils.isEmpty(lt_user) || TextUtils.isEmpty(lt_key))
return false;
// https://languagetool.org/http-api/swagger-ui/#!/default/post_words_add
// https://languagetool.org/http-api/swagger-ui/#!/default/post_words_delete
Uri.Builder builder = new Uri.Builder()
.appendQueryParameter("word", word)
.appendQueryParameter("username", lt_user)
.appendQueryParameter("apiKey", lt_key);
if (dictionary != null)
builder.appendQueryParameter("dict", dictionary);
Uri uri = Uri.parse(lt_uri).buildUpon()
.appendPath(add ? "words/add" : "words/delete")
.build();
String request = builder.build().toString().substring(1);
Log.i("LT uri=" + uri + " request=" + request);
URL url = new URL(uri.toString());
2023-12-06 14:31:05 +00:00
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
2022-11-07 09:51:34 +00:00
connection.setRequestMethod("POST");
connection.setDoOutput(true);
connection.setReadTimeout(LT_TIMEOUT * 1000);
connection.setConnectTimeout(LT_TIMEOUT * 1000);
ConnectionHelper.setUserAgent(context, connection);
connection.setRequestProperty("Accept", "application/json");
connection.setRequestProperty("Content-Length", Integer.toString(request.length()));
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.connect();
try {
connection.getOutputStream().write(request.getBytes());
checkStatus(connection);
String response = Helper.readStream(connection.getInputStream());
Log.i("LT response=" + response);
JSONObject jroot = new JSONObject(response);
return jroot.getBoolean(add ? "added" : "deleted");
} finally {
connection.disconnect();
}
}
static List<String> getWords(Context context, String[] dictionary) throws IOException, JSONException {
List<String> result = new ArrayList<>();
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String lt_user = prefs.getString("lt_user", null);
String lt_key = prefs.getString("lt_key", null);
String lt_uri = prefs.getString("lt_uri", LT_URI_PLUS);
if (TextUtils.isEmpty(lt_user) || TextUtils.isEmpty(lt_key))
return result;
// https://languagetool.org/http-api/swagger-ui/#!/default/get_words
Uri.Builder builder = Uri.parse(lt_uri).buildUpon()
.appendPath("words")
.appendQueryParameter("offset", "0")
.appendQueryParameter("limit", "500")
.appendQueryParameter("username", lt_user)
.appendQueryParameter("apiKey", lt_key);
if (dictionary != null && dictionary.length > 0)
builder.appendQueryParameter("dicts", TextUtils.join(",", dictionary));
Uri uri = builder.build();
Log.i("LT uri=" + uri);
URL url = new URL(uri.toString());
2023-12-06 14:31:05 +00:00
HttpsURLConnection connection = (HttpsURLConnection) url.openConnection();
2022-11-07 09:51:34 +00:00
connection.setRequestMethod("GET");
connection.setDoOutput(false);
connection.setReadTimeout(LT_TIMEOUT * 1000);
connection.setConnectTimeout(LT_TIMEOUT * 1000);
ConnectionHelper.setUserAgent(context, connection);
connection.setRequestProperty("Accept", "application/json");
connection.setRequestProperty("Content-Type", "application/x-www-form-urlencoded");
connection.connect();
try {
checkStatus(connection);
String response = Helper.readStream(connection.getInputStream());
Log.i("LT response=" + response);
JSONObject jroot = new JSONObject(response);
JSONArray jwords = jroot.getJSONArray("words");
for (int i = 0; i < jwords.length(); i++)
result.add(jwords.getString(i));
return result;
} finally {
connection.disconnect();
}
}
static void applySuggestions(EditText etBody, int start, int end, List<Suggestion> suggestions) {
if (etBody == null)
return;
2022-10-01 07:51:16 +00:00
Editable edit = etBody.getText();
if (edit == null)
return;
// https://developer.android.com/reference/android/text/style/SuggestionSpan
for (SuggestionSpanEx suggestion : edit.getSpans(start, end, SuggestionSpanEx.class)) {
2022-10-01 10:52:14 +00:00
Log.i("LT removing=" + suggestion);
edit.removeSpan(suggestion);
2022-10-01 07:51:16 +00:00
}
2022-10-01 10:52:14 +00:00
if (suggestions != null)
for (LanguageTool.Suggestion suggestion : suggestions) {
2023-09-11 07:10:19 +00:00
boolean misspelled = ("misspelling".equals(suggestion.issueType) ||
"typographical".equals(suggestion.issueType) ||
2023-09-11 07:10:19 +00:00
"whitespace".equals(suggestion.issueType));
2023-09-13 06:53:24 +00:00
SpannableStringBuilder ssb = new SpannableStringBuilderEx();
if (!TextUtils.isEmpty(suggestion.title))
ssb.append(suggestion.title);
if (!TextUtils.isEmpty(suggestion.description)) {
if (ssb.length() > 0)
ssb.append(": ");
ssb.append(suggestion.description);
}
if (!TextUtils.isEmpty(suggestion.issueType)) {
int len = ssb.length();
if (len > 0)
ssb.append(" (");
ssb.append(suggestion.issueType);
if (len > 0)
ssb.append(")");
}
2022-10-01 10:52:14 +00:00
SuggestionSpan span = new SuggestionSpanEx(etBody.getContext(),
2023-09-13 06:53:24 +00:00
ssb.toString(),
2023-09-11 07:10:19 +00:00
suggestion.replacements.toArray(new String[0]), misspelled);
int s = start + suggestion.offset;
int e = s + suggestion.length;
if (s < 0 || s > edit.length() || e < 0 || e > edit.length()) {
Log.w("LT " + s + "..." + e + " length=" + edit.length());
continue;
} else
Log.i("LT text='" + edit.subSequence(s, e) + "' " + suggestion);
2022-11-13 17:56:17 +00:00
edit.setSpan(span, s, e, Spanned.SPAN_EXCLUSIVE_EXCLUSIVE);
2022-10-01 10:52:14 +00:00
}
2022-10-01 07:51:16 +00:00
}
static boolean isPremium(Context context) {
SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
String lt_user = prefs.getString("lt_user", null);
String lt_key = prefs.getString("lt_key", null);
return (!TextUtils.isEmpty(lt_user) && !TextUtils.isEmpty(lt_key));
}
2023-12-06 14:31:05 +00:00
private static void checkStatus(HttpsURLConnection connection) throws IOException {
2022-11-07 09:51:34 +00:00
int status = connection.getResponseCode();
2023-12-06 14:31:05 +00:00
if (status != HttpsURLConnection.HTTP_OK) {
2022-11-07 09:51:34 +00:00
String error = "Error " + status + ": " + connection.getResponseMessage();
try {
InputStream is = connection.getErrorStream();
if (is != null)
error += "\n" + Helper.readStream(is);
} catch (Throwable ex) {
Log.w(ex);
}
Log.w("LT " + error);
2022-12-12 18:48:11 +00:00
throw new IOException(error);
2022-11-07 09:51:34 +00:00
}
}
2022-04-27 15:18:08 +00:00
static class Suggestion {
public String title; // shortMessage
public String description; // message
public int offset;
public int length;
public List<String> replacements;
public String issueType;
2022-04-27 15:18:08 +00:00
@Override
public String toString() {
return issueType + " " + title + " " + description;
2022-04-27 15:18:08 +00:00
}
}
}