mirror
/
FairEmail
mirror of https://github.com/M66B/FairEmail.git
1
0
Fork 0
Code Issues Releases Wiki Activity

OpenAI: image support

This commit is contained in:
M66B 2024-05-14 20:21:54 +02:00
parent 229a45d87b
commit 252ad4c2a5
4 changed files with 66 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

49
app/src/main/java/eu/faircode/email/FragmentDialogSummarize.java
View File

@ -23,6 +23,7 @@ import android.app.Dialog;
import android.content.Context;
import android.content.SharedPreferences;
import android.os.Bundle;
import android.text.SpannableStringBuilder;
import android.text.TextUtils;
import android.util.TypedValue;
import android.view.LayoutInflater;
@ -106,9 +107,6 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
HtmlHelper.removeSignatures(d);
d.select("blockquote").remove();
HtmlHelper.truncate(d, HtmlHelper.MAX_TRANSLATABLE_TEXT_SIZE);
String text = d.text();
if (TextUtils.isEmpty(text))
return null;
if (OpenAI.isAvailable(context)) {
String model = prefs.getString("openai_model", OpenAI.DEFAULT_MODEL);
@ -118,8 +116,46 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
List<OpenAI.Message> input = new ArrayList<>();
input.add(new OpenAI.Message(OpenAI.USER,
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, prompt)}));
input.add(new OpenAI.Message(OpenAI.USER,
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, text)}));
SpannableStringBuilder ssb = HtmlHelper.fromDocument(context, d, null, null);
DB db = DB.getInstance(context);
List<OpenAI.Content> contents = new ArrayList<>();
int start = 0;
while (start < ssb.length()) {
int end = ssb.nextSpanTransition(start, ssb.length(), ImageSpanEx.class);
String text = ssb.subSequence(start, end).toString();
Log.i("MMM " + start + "..." + end + " len=" + ssb.length() + " text=" + text.replace('\n', '|'));
contents.add(new OpenAI.Content(OpenAI.CONTENT_TEXT, text));
if (end < ssb.length()) {
ImageSpanEx[] spans = ssb.getSpans(end, end, ImageSpanEx.class);
if (spans.length == 1) {
int s = ssb.getSpanStart(spans[0]);
int e = ssb.getSpanEnd(spans[0]);
String src = spans[0].getSource();
String url = null;
if (src.startsWith("cid:")) {
String cid = '<' + src.substring(4) + '>';
EntityAttachment attachment = db.attachment().getAttachment(id, cid);
if (attachment != null && attachment.available)
try {
url = ImageHelper.getDataUri(attachment.getFile(context), attachment.type);
} catch (Throwable ex) {
Log.w(ex);
}
} else
url = src;
Log.i("MMM image " + s + "..." + e + " url=" + url);
if (url != null)
contents.add(new OpenAI.Content(OpenAI.CONTENT_IMAGE, url));
end = e;
}
}
start = end;
}
input.add(new OpenAI.Message(OpenAI.USER, contents.toArray(new OpenAI.Content[0])));
OpenAI.Message[] result =
OpenAI.completeChat(context, model, input.toArray(new OpenAI.Message[0]), temperature, 1);
@ -140,6 +176,9 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
float temperature = prefs.getFloat("gemini_temperature", Gemini.DEFAULT_TEMPERATURE);
String prompt = prefs.getString("gemini_summarize", Gemini.DEFAULT_SUMMARY_PROMPT);
String text = d.text();
if (TextUtils.isEmpty(text))
return null;
Gemini.Message message = new Gemini.Message(Gemini.USER, new String[]{prompt, text});
Gemini.Message[] result =

19
app/src/main/java/eu/faircode/email/HtmlHelper.java
View File

@ -103,9 +103,7 @@ import org.w3c.dom.stylesheets.MediaList;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
import java.net.URI;
import java.text.DateFormat;
@ -2467,21 +2465,8 @@ public class HtmlHelper {
Uri uri = FileProviderEx.getUri(context, BuildConfig.APPLICATION_ID, file, attachment.name);
img.attr("src", uri.toString());
Log.i("Inline image uri=" + uri);
} else {
try (InputStream is = new FileInputStream(file)) {
byte[] bytes = new byte[(int) file.length()];
if (is.read(bytes) != bytes.length)
throw new IOException("length");
StringBuilder sb = new StringBuilder();
sb.append("data:");
sb.append(attachment.type);
sb.append(";base64,");
sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP));
img.attr("src", sb.toString());
}
}
} else
img.attr("src", ImageHelper.getDataUri(file, attachment.type));
}
}
}

14
app/src/main/java/eu/faircode/email/ImageHelper.java
View File

@ -615,6 +615,20 @@ class ImageHelper {
return source.substring(colon + 1, semi);
}
static String getDataUri(File file, String type) throws IOException {
try (InputStream is = new FileInputStream(file)) {
byte[] bytes = Helper.readBytes(is);
StringBuilder sb = new StringBuilder();
sb.append("data:");
sb.append(type);
sb.append(";base64,");
sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP));
return sb.toString();
}
}
static ByteArrayInputStream getDataUriStream(String source) {
// "<img src=\"data:image/png;base64,iVBORw0KGgoAAA" +
// "ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4" +

7
app/src/main/java/eu/faircode/email/OpenAI.java
View File

@ -123,7 +123,12 @@ public class OpenAI {
for (Content content : message.content) {
JSONObject jcontent = new JSONObject();
jcontent.put("type", content.type);
jcontent.put(content.type, content.content);
if (CONTENT_IMAGE.equals(content.type)) {
JSONObject jimage = new JSONObject();
jimage.put("url", content.content);
jcontent.put(content.type, jimage);
} else
jcontent.put(content.type, content.content);
jcontents.put(jcontent);
}
jmessage.put("content", jcontents);