mirror/FairEmail
1
0
Fork 0
mirror of https://github.com/M66B/FairEmail.git synced 2025-03-15 08:29:24 +00:00
Code Issues Releases Wiki Activity

Reduce classifier memory usage (2)

Browse source
This commit is contained in:
M66B 2022-09-14 18:16:51 +02:00
parent 5816a998c2
commit aa117dbaa7
1 changed files with 24 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
app/src/main/java/eu/faircode/email/MessageClassifier.java
View file

@ -413,6 +413,8 @@ public class MessageClassifier {
long start = new Date().getTime();
reduce();
File file = getFile(context, false);
File backup = getFile(context, true);
backup.delete();
@ -646,12 +648,17 @@ public class MessageClassifier {
reader.endObject();
}
reduce();
loaded = true;
dirty = false;
long elapsed = new Date().getTime() - start;
Log.i("Classifier data loaded elapsed=" + elapsed);
}
private static void reduce() {
Log.i("Classifier reduce");
for (long account : wordClassFrequency.keySet()) {
Map<String, Long> total = new HashMap<>();
Map<String, Integer> count = new HashMap<>();
@ -671,20 +678,34 @@ public class MessageClassifier {
for (String word : wordClassFrequency.get(account).keySet())
for (String clazz : new ArrayList<>(wordClassFrequency.get(account).get(word).keySet())) {
int freq = wordClassFrequency.get(account).get(word).get(clazz).count;
long avg = total.get(clazz) / count.get(clazz);
if (freq < avg / 2) {
Frequency freq = wordClassFrequency.get(account).get(word).get(clazz);
if (freq.count < avg / 2) {
Log.i("Classifier dropping account=" + account +
" word=" + word + " class=" + clazz + " freq=" + freq + " avg=" + avg);
" word=" + word + " class=" + clazz + " freq=" + freq.count + " avg=" + avg);
wordClassFrequency.get(account).get(word).remove(clazz);
} else if (false) {
for (String b : new ArrayList<>(freq.before.keySet()))
if (freq.before.get(b) < freq.count / 20)
freq.before.remove(b);
for (String a : new ArrayList<>(freq.after.keySet()))
if (freq.after.get(a) < freq.count / 20)
freq.after.remove(a);
}
}
// Source 47 MB
// avg/1 = 21.3
// avg/2 = 25.5
// avg/3 = 29.0
// avg/5 = 34.6
// ba/5 = 27.2
// ba/10 = 29.3
// ba/20 = 31.5
// avg/2 + ba/20 = 10 MB
}
}