From 252ad4c2a589887ca63bc650fdd977a22ea1987c Mon Sep 17 00:00:00 2001 From: M66B Date: Tue, 14 May 2024 20:21:54 +0200 Subject: [PATCH] OpenAI: image support --- .../email/FragmentDialogSummarize.java | 49 +++++++++++++++++-- .../java/eu/faircode/email/HtmlHelper.java | 19 +------ .../java/eu/faircode/email/ImageHelper.java | 14 ++++++ .../main/java/eu/faircode/email/OpenAI.java | 7 ++- 4 files changed, 66 insertions(+), 23 deletions(-) diff --git a/app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java b/app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java index a880f29294..1285249199 100644 --- a/app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java +++ b/app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java @@ -23,6 +23,7 @@ import android.app.Dialog; import android.content.Context; import android.content.SharedPreferences; import android.os.Bundle; +import android.text.SpannableStringBuilder; import android.text.TextUtils; import android.util.TypedValue; import android.view.LayoutInflater; @@ -106,9 +107,6 @@ public class FragmentDialogSummarize extends FragmentDialogBase { HtmlHelper.removeSignatures(d); d.select("blockquote").remove(); HtmlHelper.truncate(d, HtmlHelper.MAX_TRANSLATABLE_TEXT_SIZE); - String text = d.text(); - if (TextUtils.isEmpty(text)) - return null; if (OpenAI.isAvailable(context)) { String model = prefs.getString("openai_model", OpenAI.DEFAULT_MODEL); @@ -118,8 +116,46 @@ public class FragmentDialogSummarize extends FragmentDialogBase { List input = new ArrayList<>(); input.add(new OpenAI.Message(OpenAI.USER, new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, prompt)})); - input.add(new OpenAI.Message(OpenAI.USER, - new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, text)})); + SpannableStringBuilder ssb = HtmlHelper.fromDocument(context, d, null, null); + + DB db = DB.getInstance(context); + List contents = new ArrayList<>(); + int start = 0; + while (start < ssb.length()) { + int end = ssb.nextSpanTransition(start, ssb.length(), ImageSpanEx.class); + String text = ssb.subSequence(start, end).toString(); + Log.i("MMM " + start + "..." + end + " len=" + ssb.length() + " text=" + text.replace('\n', '|')); + contents.add(new OpenAI.Content(OpenAI.CONTENT_TEXT, text)); + if (end < ssb.length()) { + ImageSpanEx[] spans = ssb.getSpans(end, end, ImageSpanEx.class); + if (spans.length == 1) { + int s = ssb.getSpanStart(spans[0]); + int e = ssb.getSpanEnd(spans[0]); + String src = spans[0].getSource(); + + String url = null; + if (src.startsWith("cid:")) { + String cid = '<' + src.substring(4) + '>'; + EntityAttachment attachment = db.attachment().getAttachment(id, cid); + if (attachment != null && attachment.available) + try { + url = ImageHelper.getDataUri(attachment.getFile(context), attachment.type); + } catch (Throwable ex) { + Log.w(ex); + } + } else + url = src; + + Log.i("MMM image " + s + "..." + e + " url=" + url); + if (url != null) + contents.add(new OpenAI.Content(OpenAI.CONTENT_IMAGE, url)); + end = e; + } + } + start = end; + } + + input.add(new OpenAI.Message(OpenAI.USER, contents.toArray(new OpenAI.Content[0]))); OpenAI.Message[] result = OpenAI.completeChat(context, model, input.toArray(new OpenAI.Message[0]), temperature, 1); @@ -140,6 +176,9 @@ public class FragmentDialogSummarize extends FragmentDialogBase { float temperature = prefs.getFloat("gemini_temperature", Gemini.DEFAULT_TEMPERATURE); String prompt = prefs.getString("gemini_summarize", Gemini.DEFAULT_SUMMARY_PROMPT); + String text = d.text(); + if (TextUtils.isEmpty(text)) + return null; Gemini.Message message = new Gemini.Message(Gemini.USER, new String[]{prompt, text}); Gemini.Message[] result = diff --git a/app/src/main/java/eu/faircode/email/HtmlHelper.java b/app/src/main/java/eu/faircode/email/HtmlHelper.java index f097a4abe4..0be8c8972c 100644 --- a/app/src/main/java/eu/faircode/email/HtmlHelper.java +++ b/app/src/main/java/eu/faircode/email/HtmlHelper.java @@ -103,9 +103,7 @@ import org.w3c.dom.stylesheets.MediaList; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; import java.io.StringReader; import java.net.URI; import java.text.DateFormat; @@ -2467,21 +2465,8 @@ public class HtmlHelper { Uri uri = FileProviderEx.getUri(context, BuildConfig.APPLICATION_ID, file, attachment.name); img.attr("src", uri.toString()); Log.i("Inline image uri=" + uri); - } else { - try (InputStream is = new FileInputStream(file)) { - byte[] bytes = new byte[(int) file.length()]; - if (is.read(bytes) != bytes.length) - throw new IOException("length"); - - StringBuilder sb = new StringBuilder(); - sb.append("data:"); - sb.append(attachment.type); - sb.append(";base64,"); - sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP)); - - img.attr("src", sb.toString()); - } - } + } else + img.attr("src", ImageHelper.getDataUri(file, attachment.type)); } } } diff --git a/app/src/main/java/eu/faircode/email/ImageHelper.java b/app/src/main/java/eu/faircode/email/ImageHelper.java index 6efa410c72..a2141753ea 100644 --- a/app/src/main/java/eu/faircode/email/ImageHelper.java +++ b/app/src/main/java/eu/faircode/email/ImageHelper.java @@ -615,6 +615,20 @@ class ImageHelper { return source.substring(colon + 1, semi); } + static String getDataUri(File file, String type) throws IOException { + try (InputStream is = new FileInputStream(file)) { + byte[] bytes = Helper.readBytes(is); + + StringBuilder sb = new StringBuilder(); + sb.append("data:"); + sb.append(type); + sb.append(";base64,"); + sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP)); + + return sb.toString(); + } + } + static ByteArrayInputStream getDataUriStream(String source) { // "