From 5fc2b08db95f7ca0b8c619a1befd93728edf129c Mon Sep 17 00:00:00 2001 From: M66B Date: Sat, 23 May 2020 16:05:29 +0200 Subject: [PATCH] Ensure envelope/body --- .../java/eu/faircode/email/ActivityEML.java | 4 +- app/src/main/java/eu/faircode/email/Core.java | 29 +++-- .../eu/faircode/email/FragmentMessages.java | 8 +- .../java/eu/faircode/email/MessageHelper.java | 121 ++++++++++++++---- .../java/eu/faircode/email/ServiceSend.java | 4 +- 5 files changed, 118 insertions(+), 48 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/ActivityEML.java b/app/src/main/java/eu/faircode/email/ActivityEML.java index bdaa58cc4d..7eb7b756fb 100644 --- a/app/src/main/java/eu/faircode/email/ActivityEML.java +++ b/app/src/main/java/eu/faircode/email/ActivityEML.java @@ -166,7 +166,7 @@ public class ActivityEML extends ActivityBase { Session isession = Session.getInstance(props, null); MimeMessage imessage = new MimeMessage(isession, is); - MessageHelper helper = new MessageHelper(imessage); + MessageHelper helper = new MessageHelper(imessage, context); result.from = MessageHelper.formatAddresses(helper.getFrom()); result.to = MessageHelper.formatAddresses(helper.getTo()); @@ -176,7 +176,7 @@ public class ActivityEML extends ActivityBase { result.sent = helper.getSent(); result.received = helper.getReceived(); result.subject = helper.getSubject(); - result.parts = helper.getMessageParts(context); + result.parts = helper.getMessageParts(); String html = result.parts.getHtml(context); if (html != null) { diff --git a/app/src/main/java/eu/faircode/email/Core.java b/app/src/main/java/eu/faircode/email/Core.java index f679bf59de..e256fe538c 100644 --- a/app/src/main/java/eu/faircode/email/Core.java +++ b/app/src/main/java/eu/faircode/email/Core.java @@ -1061,7 +1061,7 @@ class Core { boolean found = false; for (Message imessage : imessages) { - MessageHelper helper = new MessageHelper((MimeMessage) imessage); + MessageHelper helper = new MessageHelper((MimeMessage) imessage, context); String msgid; if (caps.containsKey("UIDL")) @@ -1145,7 +1145,7 @@ class Core { if (imessage == null) throw new MessageRemovedException(); - MessageHelper helper = new MessageHelper(imessage); + MessageHelper helper = new MessageHelper(imessage, context); db.message().setMessageHeaders(message.id, helper.getHeaders()); } @@ -1198,8 +1198,8 @@ class Core { if (imessage == null) throw new MessageRemovedException(); - MessageHelper helper = new MessageHelper((MimeMessage) imessage); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper helper = new MessageHelper((MimeMessage) imessage, context); + MessageHelper.MessageParts parts = helper.getMessageParts(); String body = parts.getHtml(context); File file = message.getFile(context); Helper.writeText(file, body); @@ -1232,8 +1232,8 @@ class Core { throw new MessageRemovedException(); // Get message parts - MessageHelper helper = new MessageHelper((MimeMessage) imessage); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper helper = new MessageHelper((MimeMessage) imessage, context); + MessageHelper.MessageParts parts = helper.getMessageParts(); // Download attachment parts.downloadAttachment(context, attachment); @@ -1547,7 +1547,7 @@ class Core { if (!state.isRunning()) return; - MessageHelper helper = new MessageHelper((MimeMessage) imessage); + MessageHelper helper = new MessageHelper((MimeMessage) imessage, context); String uidl = caps.containsKey("UIDL") ? ifolder.getUID(imessage) : helper.getMessageID(); @@ -1580,7 +1580,7 @@ class Core { sent = 0L; String authentication = helper.getAuthentication(); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper.MessageParts parts = helper.getMessageParts(); EntityMessage message = new EntityMessage(); message.account = folder.account; @@ -1986,10 +1986,11 @@ class Core { try { // Some providers erroneously return old messages if (full.contains(isub[j])) { - Date received = isub[j].getReceivedDate(); + MessageHelper helper = new MessageHelper((MimeMessage) isub[j], context); + long received = helper.getReceived(); boolean unseen = (sync_unseen && !isub[j].isSet(Flags.Flag.SEEN)); boolean flagged = (sync_flagged && isub[j].isSet(Flags.Flag.FLAGGED)); - if (received != null && received.getTime() < keep_time && !unseen && !flagged) { + if (received < keep_time && !unseen && !flagged) { long uid = ifolder.getUID(isub[j]); Log.i(folder.name + " Skipping old uid=" + uid + " date=" + received); ids[from + j] = null; @@ -2127,7 +2128,7 @@ class Core { throw new MessageRemovedException("Flagged deleted"); } - MessageHelper helper = new MessageHelper(imessage); + MessageHelper helper = new MessageHelper(imessage, context); boolean seen = helper.getSeen(); boolean answered = helper.getAnsered(); boolean flagged = helper.getFlagged(); @@ -2197,7 +2198,7 @@ class Core { } String authentication = helper.getAuthentication(); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper.MessageParts parts = helper.getMessageParts(); message = new EntityMessage(); message.account = folder.account; @@ -2762,8 +2763,8 @@ class Core { // fp.add(GmailFolder.FetchProfileItem.THRID); //ifolder.fetch(new Message[]{imessage}, fp); - MessageHelper helper = new MessageHelper(imessage); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper helper = new MessageHelper(imessage, context); + MessageHelper.MessageParts parts = helper.getMessageParts(); if (!message.content) { if (state.getNetworkState().isUnmetered() || diff --git a/app/src/main/java/eu/faircode/email/FragmentMessages.java b/app/src/main/java/eu/faircode/email/FragmentMessages.java index 90c00c453a..a27bcb5649 100644 --- a/app/src/main/java/eu/faircode/email/FragmentMessages.java +++ b/app/src/main/java/eu/faircode/email/FragmentMessages.java @@ -5314,8 +5314,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. Session isession = Session.getInstance(props, null); try (InputStream fis = new FileInputStream(plain)) { MimeMessage imessage = new MimeMessage(isession, fis); - MessageHelper helper = new MessageHelper(imessage); - parts = helper.getMessageParts(context); + MessageHelper helper = new MessageHelper(imessage, context); + parts = helper.getMessageParts(); } try { @@ -5917,8 +5917,8 @@ public class FragmentMessages extends FragmentBase implements SharedPreferences. Properties props = MessageHelper.getSessionProperties(); Session isession = Session.getInstance(props, null); MimeMessage imessage = new MimeMessage(isession, is); - MessageHelper helper = new MessageHelper(imessage); - MessageHelper.MessageParts parts = helper.getMessageParts(context); + MessageHelper helper = new MessageHelper(imessage, context); + MessageHelper.MessageParts parts = helper.getMessageParts(); DB db = DB.getInstance(context); try { diff --git a/app/src/main/java/eu/faircode/email/MessageHelper.java b/app/src/main/java/eu/faircode/email/MessageHelper.java index fe753a2c6e..ba9690ebd3 100644 --- a/app/src/main/java/eu/faircode/email/MessageHelper.java +++ b/app/src/main/java/eu/faircode/email/MessageHelper.java @@ -29,6 +29,7 @@ import androidx.documentfile.provider.DocumentFile; import androidx.preference.PreferenceManager; import com.sun.mail.gimap.GmailMessage; +import com.sun.mail.imap.IMAPMessage; import com.sun.mail.util.ASCIIUtility; import com.sun.mail.util.BASE64DecoderStream; import com.sun.mail.util.FolderClosedIOException; @@ -100,8 +101,12 @@ import biweekly.Biweekly; import biweekly.ICalendar; public class MessageHelper { + private boolean ensuredEnvelope = false; + private boolean ensuredBody = false; private MimeMessage imessage; + private static File cacheDir = null; + static final int SMALL_MESSAGE_SIZE = 64 * 1024; // bytes static final int DEFAULT_ATTACHMENT_DOWNLOAD_SIZE = 256 * 1024; // bytes @@ -747,7 +752,9 @@ public class MessageHelper { } } - MessageHelper(MimeMessage message) { + MessageHelper(MimeMessage message, Context context) { + if (cacheDir == null) + cacheDir = context.getCacheDir(); this.imessage = message; } @@ -779,6 +786,8 @@ public class MessageHelper { } String getMessageID() throws MessagingException { + ensureMessage(false); + // Outlook outbox -> sent String header = imessage.getHeader("X-Correlation-ID", null); if (header == null) @@ -787,6 +796,8 @@ public class MessageHelper { } String[] getReferences() throws MessagingException { + ensureMessage(false); + List result = new ArrayList<>(); String refs = imessage.getHeader("References", null); if (refs != null) @@ -847,6 +858,8 @@ public class MessageHelper { } String getDeliveredTo() throws MessagingException { + ensureMessage(false); + String header = imessage.getHeader("Delivered-To", null); if (header == null) header = imessage.getHeader("X-Delivered-To", null); @@ -861,11 +874,15 @@ public class MessageHelper { } String getInReplyTo() throws MessagingException { + ensureMessage(false); + String header = imessage.getHeader("In-Reply-To", null); return (header == null ? null : MimeUtility.unfold(header)); } String getThreadId(Context context, long account, long uid) throws MessagingException { + ensureMessage(false); + if (imessage instanceof GmailMessage) { // https://developers.google.com/gmail/imap/imap-extensions#access_to_the_gmail_thread_id_x-gm-thrid SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context); @@ -927,6 +944,8 @@ public class MessageHelper { Integer getPriority() throws MessagingException { Integer priority = null; + ensureMessage(false); + // https://docs.microsoft.com/en-us/openspecs/exchange_server_protocols/ms-oxcmail/2bb19f1b-b35e-4966-b1cb-1afd044e83ab String header = imessage.getHeader("Importance", null); if (header == null) @@ -985,15 +1004,21 @@ public class MessageHelper { } boolean getReceiptRequested() throws MessagingException { + ensureMessage(false); + return (imessage.getHeader("Return-Receipt-To") != null || imessage.getHeader("Disposition-Notification-To") != null); } Address[] getReceiptTo() throws MessagingException { + ensureMessage(false); + return getAddressHeader("Disposition-Notification-To"); } String getAuthentication() throws MessagingException { + ensureMessage(false); + String header = imessage.getHeader("Authentication-Results", null); return (header == null ? null : MimeUtility.unfold(header)); } @@ -1024,6 +1049,8 @@ public class MessageHelper { } String getReceivedFromHost() throws MessagingException { + ensureMessage(false); + String[] received = imessage.getHeader("Received"); if (received == null || received.length == 0) return null; @@ -1062,6 +1089,8 @@ public class MessageHelper { } private Address[] getAddressHeader(String name) throws MessagingException { + ensureMessage(false); + String header = imessage.getHeader(name, ","); if (header == null) return null; @@ -1121,6 +1150,8 @@ public class MessageHelper { } Address[] getListPost() throws MessagingException { + ensureMessage(false); + String list; try { // https://www.ietf.org/rfc/rfc2369.txt @@ -1159,6 +1190,8 @@ public class MessageHelper { } String getListUnsubscribe() throws MessagingException { + ensureMessage(false); + String list; try { // https://www.ietf.org/rfc/rfc2369.txt @@ -1203,6 +1236,8 @@ public class MessageHelper { } String getAutocrypt() throws MessagingException { + ensureMessage(false); + String autocrypt = imessage.getHeader("Autocrypt", null); if (autocrypt == null) return null; @@ -1211,6 +1246,8 @@ public class MessageHelper { } String getSubject() throws MessagingException { + ensureMessage(false); + String subject = imessage.getHeader("Subject", null); if (subject == null) return null; @@ -1227,11 +1264,15 @@ public class MessageHelper { } Long getSize() throws MessagingException { + ensureMessage(false); + long size = imessage.getSize(); return (size < 0 ? null : size); } long getReceived() throws MessagingException { + ensureMessage(false); + Date received = imessage.getReceivedDate(); if (received == null) received = imessage.getSentDate(); @@ -1239,11 +1280,15 @@ public class MessageHelper { } Long getSent() throws MessagingException { + ensureMessage(false); + Date date = imessage.getSentDate(); return (date == null ? null : date.getTime()); } String getHeaders() throws MessagingException { + ensureMessage(false); + StringBuilder sb = new StringBuilder(); Enumeration
headers = imessage.getAllHeaders(); while (headers.hasMoreElements()) { @@ -1817,33 +1862,10 @@ public class MessageHelper { EntityAttachment attachment; } - MessageParts getMessageParts(Context context) throws IOException, MessagingException { + MessageParts getMessageParts() throws IOException, MessagingException { MessageParts parts = new MessageParts(); - try { - imessage.getContentType(); - } catch (MessagingException ex) { - // https://javaee.github.io/javamail/FAQ#imapserverbug - if ("Failed to load IMAP envelope".equals(ex.getMessage()) || - "Unable to load BODYSTRUCTURE".equals(ex.getMessage())) { - Log.w("Fetching raw message"); - File file = File.createTempFile("serverbug", null, context.getCacheDir()); - try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) { - imessage.writeTo(os); - } - - Properties props = MessageHelper.getSessionProperties(); - Session isession = Session.getInstance(props, null); - - Log.w("Decoding raw message"); - try (InputStream is = new BufferedInputStream(new FileInputStream(file))) { - imessage = new MimeMessageEx(isession, is, imessage); - } - - file.delete(); - } else - throw ex; - } + ensureMessage(true); try { MimePart part = imessage; @@ -2044,6 +2066,53 @@ public class MessageHelper { } } + private void ensureMessage(boolean body) throws MessagingException { + if (body ? ensuredBody : ensuredEnvelope) + return; + + if (body) + ensuredBody = true; + else + ensuredEnvelope = true; + + Log.i("Ensure body=" + body); + + try { + if (imessage instanceof IMAPMessage) { + if (body) + imessage.getContentType(); // force loadBODYSTRUCTURE + else + imessage.getMessageID(); // force loadEnvelope + } + } catch (MessagingException ex) { + // https://javaee.github.io/javamail/FAQ#imapserverbug + if ("Failed to load IMAP envelope".equals(ex.getMessage()) || + "Unable to load BODYSTRUCTURE".equals(ex.getMessage())) + try { + Log.w("Fetching raw message"); + File file = File.createTempFile("serverbug", null, cacheDir); + try (OutputStream os = new BufferedOutputStream(new FileOutputStream(file))) { + imessage.writeTo(os); + } + + Properties props = MessageHelper.getSessionProperties(); + Session isession = Session.getInstance(props, null); + + Log.w("Decoding raw message"); + try (InputStream is = new BufferedInputStream(new FileInputStream(file))) { + imessage = new MimeMessageEx(isession, is, imessage); + } + + file.delete(); + } catch (IOException ex1) { + Log.e(ex1); + throw ex; + } + else + throw ex; + } + } + static String sanitizeKeyword(String keyword) { // https://tools.ietf.org/html/rfc3501 StringBuilder sb = new StringBuilder(); diff --git a/app/src/main/java/eu/faircode/email/ServiceSend.java b/app/src/main/java/eu/faircode/email/ServiceSend.java index c6b7b8e301..ef673de57b 100644 --- a/app/src/main/java/eu/faircode/email/ServiceSend.java +++ b/app/src/main/java/eu/faircode/email/ServiceSend.java @@ -430,14 +430,14 @@ public class ServiceSend extends ServiceBase { long id = message.id; imessage.saveChanges(); - MessageHelper helper = new MessageHelper(imessage); + MessageHelper helper = new MessageHelper(imessage, ServiceSend.this); if (message.uid != null) { Log.e("Outbox id=" + message.id + " uid=" + message.uid); message.uid = null; } - MessageHelper.MessageParts parts = helper.getMessageParts(this); + MessageHelper.MessageParts parts = helper.getMessageParts(); String body = parts.getHtml(this); try {