mirror
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Rewritten body part decoding

This commit is contained in:
M66B 2019-01-16 10:57:57 +00:00
parent f61a76af80
commit b8f6501088
9 changed files with 224 additions and 258 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -136,7 +136,7 @@ FairEmail uses:
* [JavaMail](https://projects.eclipse.org/projects/ee4j.javamail). Copyright (c) 1997-2018 Oracle® and/or its affiliates. All rights reserved. [GPLv2+CE license](https://javaee.github.io/javamail/JavaMail-License).
* [jsoup](https://jsoup.org/). Copyright © 2009 - 2017 Jonathan Hedley. [MIT license](https://jsoup.org/license).
* [JCharset](http://www.freeutils.net/source/jcharset/). Copyright © 2005-2015 Amichai Rothman. [GNU General Public License](http://www.freeutils.net/source/jcharset/#license)
* ~~[JCharset](http://www.freeutils.net/source/jcharset/). Copyright © 2005-2015 Amichai Rothman. [GNU General Public License](http://www.freeutils.net/source/jcharset/#license)~~
* [Android Support Library](https://developer.android.com/tools/support-library/). Copyright (C) 2011 The Android Open Source Project. [Apache license](https://android.googlesource.com/platform/frameworks/support/+/master/LICENSE.txt).
* [Android Architecture Components](https://developer.android.com/topic/libraries/architecture/). Copyright 2018 The Android Open Source Project, Inc. [Apache license](https://github.com/googlesamples/android-architecture-components/blob/master/LICENSE).
* [colorpicker](https://android.googlesource.com/platform/frameworks/opt/colorpicker). Copyright (C) 2013 The Android Open Source Project. [Apache license](https://android.googlesource.com/platform/frameworks/opt/colorpicker/+/master/src/com/android/colorpicker/ColorPickerDialog.java).

2
app/build.gradle
View File

@ -158,7 +158,7 @@ dependencies {
implementation "org.jsoup:jsoup:$jsoup_version"
// http://www.freeutils.net/source/jcharset/
implementation "net.freeutils:jcharset:$jcharset_version"
//implementation "net.freeutils:jcharset:$jcharset_version"
// http://www.xbill.org/dnsjava/
implementation "dnsjava:dnsjava:$dnsjava_version"

5
app/src/main/java/eu/faircode/email/ActivityView.java
View File

@ -1215,13 +1215,14 @@ public class ActivityView extends ActivityBilling implements FragmentManager.OnB
ByteArrayInputStream is = new ByteArrayInputStream(decrypted.toByteArray());
MimeMessage imessage = new MimeMessage(isession, is);
MessageHelper helper = new MessageHelper(imessage);
MessageHelper.MessageParts parts = helper.getMessageParts();
try {
db.beginTransaction();
// Write decrypted body
EntityMessage m = db.message().getMessage(id);
m.write(context, helper.getHtml());
m.write(context, parts.getHtml());
// Remove previously decrypted attachments
for (EntityAttachment a : attachments)
@ -1230,7 +1231,7 @@ public class ActivityView extends ActivityBilling implements FragmentManager.OnB
// Add decrypted attachments
int sequence = db.attachment().getAttachmentSequence(id);
for (EntityAttachment a : helper.getAttachments()) {
for (EntityAttachment a : parts.getAttachments()) {
a.message = id;
a.sequence = ++sequence;
a.id = db.attachment().insertAttachment(a);

55
app/src/main/java/eu/faircode/email/EntityAttachment.java
View File

@ -21,23 +21,16 @@ package eu.faircode.email;
import android.content.Context;
import java.io.BufferedOutputStream;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import javax.mail.BodyPart;
import javax.mail.MessagingException;
import javax.mail.Part;
import androidx.annotation.NonNull;
import androidx.room.Entity;
import androidx.room.ForeignKey;
import androidx.room.Ignore;
import androidx.room.Index;
import androidx.room.PrimaryKey;
@ -56,7 +49,6 @@ import static androidx.room.ForeignKey.CASCADE;
)
public class EntityAttachment {
static final String TABLE_NAME = "attachment";
static final int ATTACHMENT_BUFFER_SIZE = 8192; // bytes
static final Integer PGP_MESSAGE = 1;
static final Integer PGP_SIGNATURE = 2;
@ -78,9 +70,6 @@ public class EntityAttachment {
@NonNull
public Boolean available = false;
@Ignore
BodyPart part;
boolean isInline() {
return (disposition != null && disposition.equalsIgnoreCase(Part.INLINE));
}
@ -104,49 +93,6 @@ public class EntityAttachment {
}
}
void download(Context context, DB db) throws MessagingException, IOException {
// Build filename
File file = EntityAttachment.getFile(context, this.id);
// Download attachment
InputStream is = null;
OutputStream os = null;
try {
db.attachment().setProgress(this.id, null);
is = this.part.getInputStream();
os = new BufferedOutputStream(new FileOutputStream(file));
long size = 0;
byte[] buffer = new byte[ATTACHMENT_BUFFER_SIZE];
for (int len = is.read(buffer); len != -1; len = is.read(buffer)) {
size += len;
os.write(buffer, 0, len);
// Update progress
if (this.size != null)
db.attachment().setProgress(this.id, (int)(size * 100 / this.size));
}
// Store attachment data
db.attachment().setDownloaded(this.id, size);
Log.i("Downloaded attachment size=" + this.size);
} catch (IOException ex) {
// Reset progress on failure
db.attachment().setProgress(this.id, null);
throw ex;
} finally {
try {
if (is != null)
is.close();
} finally {
if (os != null)
os.close();
}
}
}
@Override
public boolean equals(Object obj) {
if (obj instanceof EntityAttachment) {
@ -164,5 +110,4 @@ public class EntityAttachment {
} else
return false;
}
}

5
app/src/main/java/eu/faircode/email/FragmentCompose.java
View File

@ -1284,7 +1284,7 @@ public class FragmentCompose extends FragmentBase {
is = context.getContentResolver().openInputStream(uri);
os = new BufferedOutputStream(new FileOutputStream(file));
byte[] buffer = new byte[EntityAttachment.ATTACHMENT_BUFFER_SIZE];
byte[] buffer = new byte[MessageHelper.ATTACHMENT_BUFFER_SIZE];
for (int len = is.read(buffer); len != -1; len = is.read(buffer)) {
size += len;
os.write(buffer, 0, len);
@ -1575,8 +1575,7 @@ public class FragmentCompose extends FragmentBase {
result.draft.id = db.message().insertMessage(result.draft);
result.draft.write(context, body == null ? "" : body);
db.message().setMessageContent(
result.draft.id, true, HtmlHelper.getPreview(body));
db.message().setMessageContent(result.draft.id, true, HtmlHelper.getPreview(body));
if ("new".equals(action)) {
ArrayList<Uri> uris = args.getParcelableArrayList("attachments");

347
app/src/main/java/eu/faircode/email/MessageHelper.java
View File

@ -26,12 +26,15 @@ import android.webkit.MimeTypeMap;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.UnsupportedEncodingException;
import java.nio.charset.Charset;
import java.util.ArrayList;
@ -65,6 +68,8 @@ public class MessageHelper {
private final static int FETCH_SIZE = 1024 * 1024; // bytes, default 16K
private final static int POOL_TIMEOUT = 3 * 60 * 1000; // milliseconds, default 45 sec
static final int ATTACHMENT_BUFFER_SIZE = 8192; // bytes
static Properties getSessionProperties(int auth_type, String realm, boolean insecure) {
Properties props = new Properties();
@ -526,145 +531,172 @@ public class MessageHelper {
return address.getAddress();
}
String getHtml() throws MessagingException, IOException {
return getHtml(imessage);
}
class MessageParts {
private Part plain = null;
private Part html = null;
private List<AttachmentPart> attachments = new ArrayList<>();
private static String readStream(InputStream is, String charset) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
for (int len = is.read(buffer); len != -1; len = is.read(buffer))
os.write(buffer, 0, len);
return new String(os.toByteArray(), charset);
}
String getHtml() throws MessagingException {
if (plain == null && html == null)
return null;
private static String getHtml(Part part) throws MessagingException, IOException {
String disposition;
try {
disposition = part.getDisposition();
} catch (MessagingException ex) {
Log.w(ex);
disposition = null;
}
String result;
boolean text = false;
Part part = (html == null ? plain : html);
if (!Part.ATTACHMENT.equalsIgnoreCase(disposition) &&
(part.isMimeType("text/plain") || part.isMimeType("text/html"))) {
String s;
try {
Object content = part.getContent();
try {
if (content instanceof String)
s = (String) content;
else if (content instanceof InputStream)
// Typically com.sun.mail.util.QPDecoderStream
s = readStream((InputStream) content, "UTF-8");
else
s = content.toString();
} catch (UnsupportedEncodingException ex) {
// x-binaryenc
// https://javaee.github.io/javamail/FAQ#unsupen
Log.w("Unsupported encoding: " + part.getContentType());
return readStream(part.getInputStream(), "US-ASCII");
if (content instanceof String)
result = (String) content;
else if (content instanceof InputStream)
// Typically com.sun.mail.util.QPDecoderStream
result = readStream((InputStream) content, "UTF-8");
else
result = content.toString();
} catch (Throwable ex) {
Log.w(ex);
text = true;
result = ex + "\n" + android.util.Log.getStackTraceString(ex);
}
if (part.isMimeType("text/plain") || text)
result = "<pre>" + result.replaceAll("\\r?\\n", "<br />") + "</pre>";
if (part.isMimeType("text/plain")) {
Log.i("Plain text");
return result;
} else {
Log.i("HTML text");
return result;
}
}
List<EntityAttachment> getAttachments() throws MessagingException {
List<EntityAttachment> result = new ArrayList<>();
for (AttachmentPart apart : attachments) {
ContentType ct = new ContentType(apart.part.getContentType());
String[] cid = apart.part.getHeader("Content-ID");
EntityAttachment attachment = new EntityAttachment();
attachment.name = apart.filename;
attachment.type = ct.getBaseType().toLowerCase();
attachment.disposition = apart.disposition;
attachment.size = (long) apart.part.getSize();
attachment.cid = (cid == null || cid.length == 0 ? null : cid[0]);
attachment.encryption = (apart.pgp ? EntityAttachment.PGP_MESSAGE : null);
if ("text/calendar".equalsIgnoreCase(attachment.type) && TextUtils.isEmpty(attachment.name))
attachment.name = "invite.ics";
// Try to guess a better content type
// Sometimes PDF files are sent using the wrong type
if ("application/octet-stream".equalsIgnoreCase(attachment.type)) {
String extension = Helper.getExtension(attachment.name);
if (extension != null) {
String type = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension.toLowerCase());
if (type != null) {
Log.w("Guessing file=" + attachment.name + " type=" + type);
attachment.type = type;
}
}
}
if (attachment.size < 0)
attachment.size = null;
result.add(attachment);
}
// Fix duplicate CIDs
for (int i = 0; i < result.size(); i++) {
String cid = result.get(i).cid;
if (cid != null)
for (int j = i + 1; j < result.size(); j++) {
EntityAttachment a = result.get(j);
if (cid.equals(a.cid))
a.cid = null;
}
}
return result;
}
void downloadAttachment(Context context, DB db, long id, int sequence) throws MessagingException, IOException {
// Attachments of drafts might not have been uploaded yet
if (sequence > attachments.size()) {
Log.w("Attachment unavailable sequence=" + sequence + " size=" + attachments.size());
return;
}
// Get data
AttachmentPart apart = attachments.get(sequence - 1);
long total = apart.part.getSize();
File file = EntityAttachment.getFile(context, id);
// Download attachment
OutputStream os = null;
try {
db.attachment().setProgress(id, null);
InputStream is = apart.part.getInputStream();
os = new BufferedOutputStream(new FileOutputStream(file));
long size = 0;
byte[] buffer = new byte[ATTACHMENT_BUFFER_SIZE];
for (int len = is.read(buffer); len != -1; len = is.read(buffer)) {
size += len;
os.write(buffer, 0, len);
// Update progress
if (total > 0)
db.attachment().setProgress(id, (int) (size * 100 / total));
}
// Store attachment data
db.attachment().setDownloaded(id, size);
Log.i("Downloaded attachment size=" + size);
} catch (IOException ex) {
// IOException; Unknown encoding: none
Log.w(ex);
return "<pre>" + ex + "<br />" + android.util.Log.getStackTraceString(ex) + "</pre>";
// Reset progress on failure
db.attachment().setProgress(id, null);
throw ex;
} finally {
if (os != null)
os.close();
}
if (part.isMimeType("text/plain"))
s = "<pre>" + s.replaceAll("\\r?\\n", "<br />") + "</pre>";
return s;
}
if (part.isMimeType("multipart/alternative")) {
String text = null;
try {
Multipart mp = (Multipart) part.getContent();
for (int i = 0; i < mp.getCount(); i++) {
Part bp = mp.getBodyPart(i);
if (bp.isMimeType("text/plain")) {
if (text == null)
text = getHtml(bp);
} else if (bp.isMimeType("text/html")) {
String s = getHtml(bp);
if (s != null)
return s;
} else
return getHtml(bp);
}
} catch (ParseException ex) {
// ParseException: In parameter list boundary="...">, expected parameter name, got ";"
Log.w(ex);
text = "<pre>" + ex + "<br />" + android.util.Log.getStackTraceString(ex) + "</pre>";
}
return text;
}
if (part.isMimeType("multipart/*"))
try {
Multipart mp = (Multipart) part.getContent();
for (int i = 0; i < mp.getCount(); i++) {
String s = getHtml(mp.getBodyPart(i));
if (s != null)
return s;
}
} catch (ParseException ex) {
Log.w(ex);
return "<pre>" + ex + "<br />" + android.util.Log.getStackTraceString(ex) + "</pre>";
}
return null;
}
public List<EntityAttachment> getAttachments() throws IOException, MessagingException {
List<EntityAttachment> result = new ArrayList<>();
private class AttachmentPart {
String disposition;
String filename;
boolean pgp;
Part part;
}
try {
Object content = imessage.getContent();
if (content instanceof String)
return result;
MessageParts getMessageParts() throws IOException, MessagingException {
MessageParts parts = new MessageParts();
getMessageParts(imessage, parts, false); // Can throw ParseException
return parts;
}
if (content instanceof Multipart) {
boolean pgp = false;
Multipart multipart = (Multipart) content;
for (int i = 0; i < multipart.getCount(); i++) {
BodyPart part = multipart.getBodyPart(i);
result.addAll(getAttachments(part, pgp));
ContentType ct = new ContentType(part.getContentType());
private void getMessageParts(Part part, MessageParts parts, boolean pgp) throws MessagingException, IOException {
if (part.isMimeType("multipart/*")) {
Multipart multipart = (Multipart) part.getContent();
for (int i = 0; i < multipart.getCount(); i++)
try {
Part cpart = multipart.getBodyPart(i);
getMessageParts(cpart, parts, pgp);
ContentType ct = new ContentType(cpart.getContentType());
if ("application/pgp-encrypted".equals(ct.getBaseType().toLowerCase()))
pgp = true;
} catch (ParseException ex) {
// Nested body: try to continue
// ParseException: In parameter list boundary="...">, expected parameter name, got ";"
Log.w(ex);
}
}
} catch (IOException ex) {
if (ex.getCause() instanceof MessagingException)
Log.w(ex);
else
throw ex;
} catch (ParseException ex) {
Log.w(ex);
}
return result;
}
private static List<EntityAttachment> getAttachments(BodyPart part, boolean pgp) throws
IOException, MessagingException {
List<EntityAttachment> result = new ArrayList<>();
Object content;
try {
content = part.getContent();
} catch (UnsupportedEncodingException ex) {
Log.w("attachment content type=" + part.getContentType());
content = part.getInputStream();
} catch (ParseException ex) {
Log.w(ex);
content = null;
}
if (content instanceof InputStream || content instanceof String) {
} else {
// https://www.iana.org/assignments/cont-disp/cont-disp.xhtml
String disposition;
try {
@ -682,54 +714,35 @@ public class MessageHelper {
filename = null;
}
if (Part.ATTACHMENT.equalsIgnoreCase(disposition) ||
!(part.isMimeType("text/plain") || part.isMimeType("text/html")) ||
!TextUtils.isEmpty(filename)) {
ContentType ct = new ContentType(part.getContentType());
String[] cid = part.getHeader("Content-ID");
Log.i("Part" +
" disposition=" + disposition +
" filename=" + filename +
" content type=" + part.getContentType());
EntityAttachment attachment = new EntityAttachment();
attachment.name = filename;
attachment.type = ct.getBaseType().toLowerCase();
attachment.disposition = disposition;
attachment.size = (long) part.getSize();
attachment.cid = (cid == null || cid.length == 0 ? null : cid[0]);
attachment.encryption = (pgp ? EntityAttachment.PGP_MESSAGE : null);
attachment.part = part;
if (TextUtils.isEmpty(attachment.name) && "text/calendar".equals(attachment.type))
attachment.name = "invite.ics";
// Try to guess a better content type
// Sometimes PDF files are sent using the wrong type
if ("application/octet-stream".equals(attachment.type)) {
String extension = Helper.getExtension(attachment.name);
if (extension != null) {
String type = MimeTypeMap.getSingleton().getMimeTypeFromExtension(extension.toLowerCase());
if (type != null) {
Log.w("Guessing file=" + attachment.name + " type=" + type);
attachment.type = type;
}
}
}
if (attachment.size < 0)
attachment.size = null;
result.add(attachment);
}
} else if (content instanceof Multipart) {
Multipart multipart = (Multipart) content;
for (int i = 0; i < multipart.getCount(); i++) {
BodyPart cpart = multipart.getBodyPart(i);
result.addAll(getAttachments(cpart, pgp));
ContentType ct = new ContentType(cpart.getContentType());
if ("application/pgp-encrypted".equals(ct.getBaseType().toLowerCase()))
pgp = true;
if (!Part.ATTACHMENT.equalsIgnoreCase(disposition) &&
((parts.plain == null && part.isMimeType("text/plain")) ||
(parts.html == null && part.isMimeType("text/html")))) {
if (part.isMimeType("text/plain"))
parts.plain = part;
else
parts.html = part;
} else {
AttachmentPart apart = new AttachmentPart();
apart.disposition = disposition;
apart.filename = filename;
apart.pgp = pgp;
apart.part = part;
parts.attachments.add(apart);
}
}
}
return result;
private static String readStream(InputStream is, String charset) throws IOException {
ByteArrayOutputStream os = new ByteArrayOutputStream();
byte[] buffer = new byte[4096];
for (int len = is.read(buffer); len != -1; len = is.read(buffer))
os.write(buffer, 0, len);
return new String(os.toByteArray(), charset);
}
static boolean equal(Address[] a1, Address[] a2) {

41
app/src/main/java/eu/faircode/email/ServiceSynchronize.java
View File

@ -1932,7 +1932,8 @@ public class ServiceSynchronize extends LifecycleService {
throw new MessageRemovedException();
MessageHelper helper = new MessageHelper((MimeMessage) imessage);
String body = helper.getHtml();
MessageHelper.MessageParts parts = helper.getMessageParts();
String body = parts.getHtml();
String preview = HtmlHelper.getPreview(body);
message.write(this, body);
db.message().setMessageContent(message.id, true, preview);
@ -1954,14 +1955,8 @@ public class ServiceSynchronize extends LifecycleService {
// Download attachment
MessageHelper helper = new MessageHelper((MimeMessage) imessage);
List<EntityAttachment> attachments = helper.getAttachments();
if (sequence - 1 < attachments.size()) {
attachment.part = attachments.get(sequence - 1).part;
attachment.download(this, db);
} else {
db.attachment().setProgress(attachment.id, null);
throw new IllegalArgumentException("Attachment not found seq=" + sequence + " size=" + attachments.size());
}
MessageHelper.MessageParts parts = helper.getMessageParts();
parts.downloadAttachment(this, db, attachment.id, sequence);
}
private void synchronizeFolders(EntityAccount account, IMAPStore istore, ServiceState state) throws MessagingException {
@ -2480,15 +2475,11 @@ public class ServiceSynchronize extends LifecycleService {
Log.i(folder.name + " added id=" + message.id + " uid=" + message.uid);
int sequence = 1;
for (EntityAttachment attachment : helper.getAttachments()) {
MessageHelper.MessageParts parts = helper.getMessageParts();
for (EntityAttachment attachment : parts.getAttachments()) {
Log.i(folder.name + " attachment seq=" + sequence +
" name=" + attachment.name + " type=" + attachment.type +
" cid=" + attachment.cid + " pgp=" + attachment.encryption);
if (!TextUtils.isEmpty(attachment.cid) &&
db.attachment().getAttachment(message.id, attachment.cid) != null) {
Log.i("Skipping duplicated CID");
continue;
}
attachment.message = message.id;
attachment.sequence = sequence++;
attachment.id = db.attachment().insertAttachment(attachment);
@ -2600,29 +2591,21 @@ public class ServiceSynchronize extends LifecycleService {
ifolder.fetch(new Message[]{imessage}, fp);
}
MessageHelper.MessageParts parts = helper.getMessageParts();
if (!message.content)
if (!metered || (message.size != null && message.size < maxSize)) {
String body = helper.getHtml();
String body = parts.getHtml();
message.write(context, body);
db.message().setMessageContent(
message.id, true, HtmlHelper.getPreview(body));
db.message().setMessageContent(message.id, true, HtmlHelper.getPreview(body));
Log.i(folder.name + " downloaded message id=" + message.id + " size=" + message.size);
}
List<EntityAttachment> iattachments = null;
for (int i = 0; i < attachments.size(); i++) {
EntityAttachment attachment = attachments.get(i);
if (!attachment.available)
if (!metered || (attachment.size != null && attachment.size < maxSize)) {
if (iattachments == null)
iattachments = helper.getAttachments();
// Attachments of drafts might not have been uploaded yet
if (i < iattachments.size()) {
attachment.part = iattachments.get(i).part;
attachment.download(context, db);
Log.i(folder.name + " downloaded message id=" + message.id + " attachment=" + attachment.name + " size=" + message.size);
}
}
if (!metered || (attachment.size != null && attachment.size < maxSize))
parts.downloadAttachment(context, db, attachment.id, attachment.sequence);
}
}
}

24
app/src/main/java/eu/faircode/email/UnknownCharsetProvider.java Normal file
View File

@ -0,0 +1,24 @@
package eu.faircode.email;
import java.nio.charset.Charset;
import java.nio.charset.spi.CharsetProvider;
import java.util.Collections;
import java.util.Iterator;
public class UnknownCharsetProvider extends CharsetProvider {
@Override
public Iterator<Charset> charsets() {
return Collections.emptyIterator();
}
@Override
public Charset charsetForName(String name) {
// x-binaryenc
// UseInqueCodePage
// none
// unknown-8bit
// https://javaee.github.io/javamail/FAQ#unsupen
Log.w("Unknown charset=" + name);
return Charset.forName("US-ASCII");
}
}

1
app/src/main/resources/META-INF/services/java.nio.charset.spi.CharsetProvider Normal file
View File

@ -0,0 +1 @@
eu.faircode.email.UnknownCharsetProvider