From faa2a334f09d8dc8826cb1804fd4aa8c8d8f389f Mon Sep 17 00:00:00 2001 From: M66B Date: Wed, 3 Nov 2021 08:54:22 +0100 Subject: [PATCH] Limit sample size for conversation actions --- .../java/eu/faircode/email/TextHelper.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/TextHelper.java b/app/src/main/java/eu/faircode/email/TextHelper.java index 20a53f9ec8..3a7f026474 100644 --- a/app/src/main/java/eu/faircode/email/TextHelper.java +++ b/app/src/main/java/eu/faircode/email/TextHelper.java @@ -44,9 +44,10 @@ import java.util.Locale; import java.util.Set; public class TextHelper { - private static final int MAX_SAMPLE_SIZE = 8192; - private static final float MIN_PROBABILITY = 0.80f; + private static final int MAX_DETECT_SAMPLE_SIZE = 8192; + private static final float MIN_DETECT_PROBABILITY = 0.80f; private static final String TRANSLITERATOR = "Any-Latin; Latin-ASCII"; + private static final int MAX_CONVERSATION_SAMPLE_SIZE = 8192; static { System.loadLibrary("fairemail"); @@ -63,11 +64,11 @@ public class TextHelper { byte[] octets = text.getBytes(); byte[] sample; - if (octets.length < MAX_SAMPLE_SIZE) + if (octets.length < MAX_DETECT_SAMPLE_SIZE) sample = octets; else { - sample = new byte[MAX_SAMPLE_SIZE]; - System.arraycopy(octets, 0, sample, 0, MAX_SAMPLE_SIZE); + sample = new byte[MAX_DETECT_SAMPLE_SIZE]; + System.arraycopy(octets, 0, sample, 0, MAX_DETECT_SAMPLE_SIZE); } long start = new Date().getTime(); @@ -76,7 +77,7 @@ public class TextHelper { long elapse = new Date().getTime() - start; Log.i("cld3 language=" + result + " elapse=" + elapse); - if (result.probability < MIN_PROBABILITY) + if (result.probability < MIN_DETECT_PROBABILITY) return null; try { @@ -139,10 +140,14 @@ public class TextHelper { .atZone(ZoneId.systemDefault()); List input = new ArrayList<>(); for (String text : texts) - input.add(new ConversationActions.Message.Builder(author) - .setReferenceTime(dt) - .setText(text) - .build()); + if (!TextUtils.isEmpty(text)) { + if (text.length() > MAX_CONVERSATION_SAMPLE_SIZE) + text = text.substring(0, MAX_CONVERSATION_SAMPLE_SIZE); + input.add(new ConversationActions.Message.Builder(author) + .setReferenceTime(dt) + .setText(text) + .build()); + } Set excluded = new HashSet<>(Arrays.asList( ConversationAction.TYPE_OPEN_URL,