mirror of https://github.com/M66B/FairEmail.git
OpenAI: image support
This commit is contained in:
parent
229a45d87b
commit
252ad4c2a5
|
@ -23,6 +23,7 @@ import android.app.Dialog;
|
||||||
import android.content.Context;
|
import android.content.Context;
|
||||||
import android.content.SharedPreferences;
|
import android.content.SharedPreferences;
|
||||||
import android.os.Bundle;
|
import android.os.Bundle;
|
||||||
|
import android.text.SpannableStringBuilder;
|
||||||
import android.text.TextUtils;
|
import android.text.TextUtils;
|
||||||
import android.util.TypedValue;
|
import android.util.TypedValue;
|
||||||
import android.view.LayoutInflater;
|
import android.view.LayoutInflater;
|
||||||
|
@ -106,9 +107,6 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
|
||||||
HtmlHelper.removeSignatures(d);
|
HtmlHelper.removeSignatures(d);
|
||||||
d.select("blockquote").remove();
|
d.select("blockquote").remove();
|
||||||
HtmlHelper.truncate(d, HtmlHelper.MAX_TRANSLATABLE_TEXT_SIZE);
|
HtmlHelper.truncate(d, HtmlHelper.MAX_TRANSLATABLE_TEXT_SIZE);
|
||||||
String text = d.text();
|
|
||||||
if (TextUtils.isEmpty(text))
|
|
||||||
return null;
|
|
||||||
|
|
||||||
if (OpenAI.isAvailable(context)) {
|
if (OpenAI.isAvailable(context)) {
|
||||||
String model = prefs.getString("openai_model", OpenAI.DEFAULT_MODEL);
|
String model = prefs.getString("openai_model", OpenAI.DEFAULT_MODEL);
|
||||||
|
@ -118,8 +116,46 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
|
||||||
List<OpenAI.Message> input = new ArrayList<>();
|
List<OpenAI.Message> input = new ArrayList<>();
|
||||||
input.add(new OpenAI.Message(OpenAI.USER,
|
input.add(new OpenAI.Message(OpenAI.USER,
|
||||||
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, prompt)}));
|
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, prompt)}));
|
||||||
input.add(new OpenAI.Message(OpenAI.USER,
|
SpannableStringBuilder ssb = HtmlHelper.fromDocument(context, d, null, null);
|
||||||
new OpenAI.Content[]{new OpenAI.Content(OpenAI.CONTENT_TEXT, text)}));
|
|
||||||
|
DB db = DB.getInstance(context);
|
||||||
|
List<OpenAI.Content> contents = new ArrayList<>();
|
||||||
|
int start = 0;
|
||||||
|
while (start < ssb.length()) {
|
||||||
|
int end = ssb.nextSpanTransition(start, ssb.length(), ImageSpanEx.class);
|
||||||
|
String text = ssb.subSequence(start, end).toString();
|
||||||
|
Log.i("MMM " + start + "..." + end + " len=" + ssb.length() + " text=" + text.replace('\n', '|'));
|
||||||
|
contents.add(new OpenAI.Content(OpenAI.CONTENT_TEXT, text));
|
||||||
|
if (end < ssb.length()) {
|
||||||
|
ImageSpanEx[] spans = ssb.getSpans(end, end, ImageSpanEx.class);
|
||||||
|
if (spans.length == 1) {
|
||||||
|
int s = ssb.getSpanStart(spans[0]);
|
||||||
|
int e = ssb.getSpanEnd(spans[0]);
|
||||||
|
String src = spans[0].getSource();
|
||||||
|
|
||||||
|
String url = null;
|
||||||
|
if (src.startsWith("cid:")) {
|
||||||
|
String cid = '<' + src.substring(4) + '>';
|
||||||
|
EntityAttachment attachment = db.attachment().getAttachment(id, cid);
|
||||||
|
if (attachment != null && attachment.available)
|
||||||
|
try {
|
||||||
|
url = ImageHelper.getDataUri(attachment.getFile(context), attachment.type);
|
||||||
|
} catch (Throwable ex) {
|
||||||
|
Log.w(ex);
|
||||||
|
}
|
||||||
|
} else
|
||||||
|
url = src;
|
||||||
|
|
||||||
|
Log.i("MMM image " + s + "..." + e + " url=" + url);
|
||||||
|
if (url != null)
|
||||||
|
contents.add(new OpenAI.Content(OpenAI.CONTENT_IMAGE, url));
|
||||||
|
end = e;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
start = end;
|
||||||
|
}
|
||||||
|
|
||||||
|
input.add(new OpenAI.Message(OpenAI.USER, contents.toArray(new OpenAI.Content[0])));
|
||||||
|
|
||||||
OpenAI.Message[] result =
|
OpenAI.Message[] result =
|
||||||
OpenAI.completeChat(context, model, input.toArray(new OpenAI.Message[0]), temperature, 1);
|
OpenAI.completeChat(context, model, input.toArray(new OpenAI.Message[0]), temperature, 1);
|
||||||
|
@ -140,6 +176,9 @@ public class FragmentDialogSummarize extends FragmentDialogBase {
|
||||||
float temperature = prefs.getFloat("gemini_temperature", Gemini.DEFAULT_TEMPERATURE);
|
float temperature = prefs.getFloat("gemini_temperature", Gemini.DEFAULT_TEMPERATURE);
|
||||||
String prompt = prefs.getString("gemini_summarize", Gemini.DEFAULT_SUMMARY_PROMPT);
|
String prompt = prefs.getString("gemini_summarize", Gemini.DEFAULT_SUMMARY_PROMPT);
|
||||||
|
|
||||||
|
String text = d.text();
|
||||||
|
if (TextUtils.isEmpty(text))
|
||||||
|
return null;
|
||||||
Gemini.Message message = new Gemini.Message(Gemini.USER, new String[]{prompt, text});
|
Gemini.Message message = new Gemini.Message(Gemini.USER, new String[]{prompt, text});
|
||||||
|
|
||||||
Gemini.Message[] result =
|
Gemini.Message[] result =
|
||||||
|
|
|
@ -103,9 +103,7 @@ import org.w3c.dom.stylesheets.MediaList;
|
||||||
import java.io.ByteArrayInputStream;
|
import java.io.ByteArrayInputStream;
|
||||||
import java.io.ByteArrayOutputStream;
|
import java.io.ByteArrayOutputStream;
|
||||||
import java.io.File;
|
import java.io.File;
|
||||||
import java.io.FileInputStream;
|
|
||||||
import java.io.IOException;
|
import java.io.IOException;
|
||||||
import java.io.InputStream;
|
|
||||||
import java.io.StringReader;
|
import java.io.StringReader;
|
||||||
import java.net.URI;
|
import java.net.URI;
|
||||||
import java.text.DateFormat;
|
import java.text.DateFormat;
|
||||||
|
@ -2467,21 +2465,8 @@ public class HtmlHelper {
|
||||||
Uri uri = FileProviderEx.getUri(context, BuildConfig.APPLICATION_ID, file, attachment.name);
|
Uri uri = FileProviderEx.getUri(context, BuildConfig.APPLICATION_ID, file, attachment.name);
|
||||||
img.attr("src", uri.toString());
|
img.attr("src", uri.toString());
|
||||||
Log.i("Inline image uri=" + uri);
|
Log.i("Inline image uri=" + uri);
|
||||||
} else {
|
} else
|
||||||
try (InputStream is = new FileInputStream(file)) {
|
img.attr("src", ImageHelper.getDataUri(file, attachment.type));
|
||||||
byte[] bytes = new byte[(int) file.length()];
|
|
||||||
if (is.read(bytes) != bytes.length)
|
|
||||||
throw new IOException("length");
|
|
||||||
|
|
||||||
StringBuilder sb = new StringBuilder();
|
|
||||||
sb.append("data:");
|
|
||||||
sb.append(attachment.type);
|
|
||||||
sb.append(";base64,");
|
|
||||||
sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP));
|
|
||||||
|
|
||||||
img.attr("src", sb.toString());
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -615,6 +615,20 @@ class ImageHelper {
|
||||||
return source.substring(colon + 1, semi);
|
return source.substring(colon + 1, semi);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static String getDataUri(File file, String type) throws IOException {
|
||||||
|
try (InputStream is = new FileInputStream(file)) {
|
||||||
|
byte[] bytes = Helper.readBytes(is);
|
||||||
|
|
||||||
|
StringBuilder sb = new StringBuilder();
|
||||||
|
sb.append("data:");
|
||||||
|
sb.append(type);
|
||||||
|
sb.append(";base64,");
|
||||||
|
sb.append(Base64.encodeToString(bytes, Base64.NO_WRAP));
|
||||||
|
|
||||||
|
return sb.toString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
static ByteArrayInputStream getDataUriStream(String source) {
|
static ByteArrayInputStream getDataUriStream(String source) {
|
||||||
// "<img src=\"data:image/png;base64,iVBORw0KGgoAAA" +
|
// "<img src=\"data:image/png;base64,iVBORw0KGgoAAA" +
|
||||||
// "ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4" +
|
// "ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4" +
|
||||||
|
|
|
@ -123,7 +123,12 @@ public class OpenAI {
|
||||||
for (Content content : message.content) {
|
for (Content content : message.content) {
|
||||||
JSONObject jcontent = new JSONObject();
|
JSONObject jcontent = new JSONObject();
|
||||||
jcontent.put("type", content.type);
|
jcontent.put("type", content.type);
|
||||||
jcontent.put(content.type, content.content);
|
if (CONTENT_IMAGE.equals(content.type)) {
|
||||||
|
JSONObject jimage = new JSONObject();
|
||||||
|
jimage.put("url", content.content);
|
||||||
|
jcontent.put(content.type, jimage);
|
||||||
|
} else
|
||||||
|
jcontent.put(content.type, content.content);
|
||||||
jcontents.put(jcontent);
|
jcontents.put(jcontent);
|
||||||
}
|
}
|
||||||
jmessage.put("content", jcontents);
|
jmessage.put("content", jcontents);
|
||||||
|
|
Loading…
Reference in New Issue