FairEmail/app/src/main/java/eu/faircode/email/BoundaryCallbackMessages.java

package eu.faircode.email;

/*
    This file is part of FairEmail.

    FairEmail is free software: you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation, either version 3 of the License, or
    (at your option) any later version.

    FairEmail is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.

    You should have received a copy of the GNU General Public License
    along with FairEmail.  If not, see <http://www.gnu.org/licenses/>.

    Copyright 2018-2024 by Marcel Bokhorst (M66B)
*/

import android.content.Context;
import android.content.SharedPreferences;
import android.database.sqlite.SQLiteDatabase;
import android.text.TextUtils;

import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.paging.PagedList;
import androidx.preference.PreferenceManager;

import com.sun.mail.gimap.GmailFolder;
import com.sun.mail.iap.Argument;
import com.sun.mail.iap.ProtocolException;
import com.sun.mail.iap.Response;
import com.sun.mail.imap.IMAPFolder;
import com.sun.mail.imap.IMAPStore;
import com.sun.mail.imap.protocol.IMAPProtocol;
import com.sun.mail.imap.protocol.IMAPResponse;
import com.sun.mail.imap.protocol.SearchSequence;
import com.sun.mail.util.MessageRemovedIOException;

import org.json.JSONArray;
import org.json.JSONException;
import org.json.JSONObject;
import org.jsoup.nodes.Document;

import java.io.File;
import java.io.IOException;
import java.io.Serializable;
import java.lang.reflect.Constructor;
import java.nio.charset.StandardCharsets;
import java.text.Normalizer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Calendar;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;

import javax.mail.Address;
import javax.mail.FetchProfile;
import javax.mail.Flags;
import javax.mail.Folder;
import javax.mail.FolderClosedException;
import javax.mail.Message;
import javax.mail.MessageRemovedException;
import javax.mail.MessagingException;
import javax.mail.ReadOnlyFolderException;
import javax.mail.UIDFolder;
import javax.mail.internet.MimeMessage;
import javax.mail.search.AndTerm;
import javax.mail.search.BodyTerm;
import javax.mail.search.ComparisonTerm;
import javax.mail.search.FlagTerm;
import javax.mail.search.FromStringTerm;
import javax.mail.search.NotTerm;
import javax.mail.search.OrTerm;
import javax.mail.search.ReceivedDateTerm;
import javax.mail.search.RecipientStringTerm;
import javax.mail.search.SearchTerm;
import javax.mail.search.SizeTerm;
import javax.mail.search.SubjectTerm;

public class BoundaryCallbackMessages extends PagedList.BoundaryCallback<TupleMessageEx> {
    private Context context;
    private AdapterMessage.ViewType viewType;
    private Long account;
    private Long folder;
    private boolean server;
    private SearchCriteria criteria;
    private int pageSize;

    private IBoundaryCallbackMessages intf;

    private State state;

    private static ExecutorService executor = Helper.getBackgroundExecutor(1, "boundary");

    private static final int SEARCH_LIMIT_DEVICE = 1000;

    interface IBoundaryCallbackMessages {
        void onLoading();

        void onLoaded(int found);

        void onWarning(String message);

        void onException(@NonNull Throwable ex);
    }

    BoundaryCallbackMessages(
            Context context,
            AdapterMessage.ViewType viewType, long account, long folder,
            boolean server, SearchCriteria criteria,
            int pageSize) {
        this.context = context.getApplicationContext();
        this.viewType = viewType;
        this.account = (account < 0 ? null : account);
        this.folder = (folder < 0 ? null : folder);
        this.server = server;
        this.criteria = criteria;
        this.pageSize = pageSize;
    }

    State setCallback(IBoundaryCallbackMessages intf) {
        Log.i("Boundary callback=" + intf);
        if (Objects.equals(intf, this.intf))
            return this.state;

        this.intf = intf;
        this.state = new State();

        return this.state;
    }

    @Override
    public void onZeroItemsLoaded() {
        Log.i("Boundary zero loaded");
        queue_load(state);
    }

    @Override
    public void onItemAtEndLoaded(@NonNull final TupleMessageEx itemAtEnd) {
        Log.i("Boundary at end id=" + itemAtEnd.id);
        queue_load(state);
    }

    void retry() {
        executor.submit(new Runnable() {
            @Override
            public void run() {
                close(state, true);
            }
        });
        queue_load(state);
    }

    private void queue_load(final State state) {
        if (state.queued.get() > 1) {
            Log.i("Boundary queued =" + state.queued.get());
            return;
        }
        state.queued.incrementAndGet();
        Log.i("Boundary queued +" + state.queued.get());

        executor.submit(new Runnable() {
            @Override
            public void run() {
                Helper.gc();

                int free = Log.getFreeMemMb();
                Map<String, String> crumb = new HashMap<>();
                crumb.put("queued", Integer.toString(state.queued.get()));
                Log.breadcrumb("Boundary run", crumb);

                Log.i("Boundary run free=" + free);

                int found = 0;
                try {
                    if (state.destroyed || state.error) {
                        Log.i("Boundary was destroyed");
                        return;
                    }
                    if (!Objects.equals(state, BoundaryCallbackMessages.this.state)) {
                        Log.i("Boundary changed state");
                        return;
                    }

                    ApplicationEx.getMainHandler().post(new Runnable() {
                        @Override
                        public void run() {
                            if (intf != null)
                                intf.onLoading();
                        }
                    });
                    if (server)
                        try {
                            found = load_server(state);
                        } catch (Throwable ex) {
                            if (state.error || ex instanceof IllegalArgumentException)
                                throw ex;

                            Log.w("Boundary", ex);
                            close(state, true);

                            // Retry
                            found = load_server(state);
                        }
                    else
                        found = load_device(state);
                } catch (final Throwable ex) {
                    state.error = true;
                    Log.e("Boundary", ex);
                    ApplicationEx.getMainHandler().post(new Runnable() {
                        @Override
                        public void run() {
                            if (intf != null)
                                intf.onException(ex);
                        }
                    });
                } finally {
                    state.queued.decrementAndGet();
                    Log.i("Boundary queued -" + state.queued.get());
                    Helper.gc();

                    crumb.put("queued", Integer.toString(state.queued.get()));
                    Log.breadcrumb("Boundary done", crumb);

                    final int f = found;
                    ApplicationEx.getMainHandler().post(new Runnable() {
                        @Override
                        public void run() {
                            if (intf != null)
                                intf.onLoaded(f);
                        }
                    });
                }
            }
        });
    }

    private int load_device(State state) {
        DB db = DB.getInstance(context);

        Log.i("Boundary device" +
                " index=" + state.index +
                " matches=" + (state.matches == null ? null : state.matches.size()));

        long[] exclude = new long[0];
        if (folder == null) {
            List<Long> folders = new ArrayList<>();
            if (!criteria.in_trash) {
                List<EntityFolder> trash = db.folder().getFoldersByType(EntityFolder.TRASH);
                if (trash != null)
                    for (EntityFolder folder : trash)
                        folders.add(folder.id);
            }
            if (!criteria.in_junk) {
                List<EntityFolder> junk = db.folder().getFoldersByType(EntityFolder.JUNK);
                if (junk != null)
                    for (EntityFolder folder : junk)
                        folders.add(folder.id);
            }
            exclude = Helper.toLongArray(folders);
        }

        int found = 0;

        List<String> word = new ArrayList<>();
        List<String> plus = new ArrayList<>();
        List<String> minus = new ArrayList<>();
        if (criteria.query != null) {
            for (String w : criteria.query.trim().split("\\s+"))
                if (w.length() > 1 && w.startsWith("+"))
                    plus.add(w.substring(1));
                else if (w.length() > 1 && w.startsWith("-"))
                    minus.add(w.substring(1));
                else
                    word.add(w);
            if (word.size() == 0 && plus.size() > 0)
                word.add(plus.get(0));
        }

        if (criteria.fts && word.size() > 0 && !criteria.in_headers && !criteria.in_html) {
            if (state.ids == null) {
                SQLiteDatabase sdb = Fts4DbHelper.getInstance(context);
                state.ids = Fts4DbHelper.match(sdb, account, folder, exclude, criteria, TextUtils.join(" ", word));
                EntityLog.log(context, "Boundary FTS" +
                        " account=" + account +
                        " folder=" + folder +
                        " criteria=" + criteria +
                        " ids=" + state.ids.size());
            }

            List<Long> excluded = Helper.fromLongArray(exclude);

            try {
                db.beginTransaction();

                for (; state.index < state.ids.size() && found < pageSize && !state.destroyed; state.index++) {
                    long id = state.ids.get(state.index);
                    EntityMessage message = db.message().getMessage(id);
                    if (message == null || message.ui_hide)
                        continue;

                    if (excluded.contains(message.folder))
                        continue;

                    if (!matchMessage(context, message, criteria, false))
                        continue;

                    found += db.message().setMessageFound(message.id, true);
                    Log.i("Boundary matched=" + message.id + " found=" + found);
                }
                db.setTransactionSuccessful();

            } finally {
                db.endTransaction();
            }

            return found;
        }

        while (found < pageSize && !state.destroyed) {
            if (state.matches == null ||
                    (state.matches.size() > 0 && state.index >= state.matches.size())) {
                String query = (word.size() == 0 ? null : '%' + TextUtils.join("%", word) + '%');
                state.matches = db.message().matchMessages(
                        account, folder, exclude,
                        query,
                        //criteria.in_senders,
                        //criteria.in_recipients,
                        //criteria.in_subject,
                        //criteria.in_keywords,
                        //criteria.in_message,
                        //criteria.in_notes,
                        //criteria.in_headers,
                        criteria.with_unseen,
                        criteria.with_flagged,
                        criteria.with_hidden,
                        criteria.with_encrypted,
                        criteria.with_attachments,
                        criteria.with_notes,
                        criteria.with_types == null ? 0 : criteria.with_types.length,
                        criteria.with_types == null ? new String[]{} : criteria.with_types,
                        criteria.with_size,
                        criteria.after,
                        criteria.before,
                        SEARCH_LIMIT_DEVICE, state.offset);
                EntityLog.log(context, "Boundary device" +
                        " account=" + account +
                        " folder=" + folder +
                        " criteria=" + criteria +
                        " query=" + query +
                        " offset=" + state.offset +
                        " size=" + state.matches.size());
                state.offset += Math.min(state.matches.size(), SEARCH_LIMIT_DEVICE);
                state.index = 0;
            }

            if (state.matches.size() == 0)
                break;

            for (int i = state.index; i < state.matches.size() && found < pageSize && !state.destroyed; i++) {
                state.index = i + 1;

                TupleMatch match = state.matches.get(i);
                boolean matched = (criteria.query == null || Boolean.TRUE.equals(match.matched));

                if (!matched) {
                    EntityMessage message = db.message().getMessage(match.id);
                    if (message != null && !message.ui_hide)
                        matched = matchMessage(context, message, criteria, true);
                }

                if (matched) {
                    found += db.message().setMessageFound(match.id, true);
                    Log.i("Boundary matched=" + match.id + " found=" + found);
                }
            }
        }

        Log.i("Boundary device done" +
                " found=" + found + "/" + pageSize +
                " destroyed=" + state.destroyed +
                " memory=" + Log.getFreeMemMb());
        return found;
    }

    private int load_server(final State state) throws MessagingException, ProtocolException, IOException {
        DB db = DB.getInstance(context);

        SharedPreferences prefs = PreferenceManager.getDefaultSharedPreferences(context);
        boolean debug = prefs.getBoolean("debug", false);

        final EntityFolder browsable = db.folder().getBrowsableFolder(folder, criteria != null);
        if (browsable == null || !browsable.selectable || browsable.local) {
            Log.i("Boundary not browsable=" + (folder != null));
            return 0;
        }

        EntityAccount account = db.account().getAccount(browsable.account);
        if (account == null || account.protocol != EntityAccount.TYPE_IMAP)
            return 0;

        if (state.imessages == null)
            try {
                // Check connectivity
                if (!ConnectionHelper.getNetworkState(context).isSuitable())
                    throw new IllegalStateException(context.getString(R.string.title_no_internet));

                EntityLog.log(context, "Boundary server connecting account=" + account.name);
                state.iservice = new EmailService(
                        context, account.getProtocol(), account.realm, account.encryption, account.insecure, account.unicode,
                        EmailService.PURPOSE_SEARCH, debug || BuildConfig.DEBUG);
                state.iservice.setPartialFetch(account.partial_fetch);
                state.iservice.setRawFetch(account.raw_fetch);
                state.iservice.setIgnoreBodyStructureSize(account.ignore_size);
                state.iservice.connect(account);

                EntityLog.log(context, "Boundary server opening folder=" + browsable.name);
                state.ifolder = (IMAPFolder) state.iservice.getStore().getFolder(browsable.name);
                try {
                    state.ifolder.open(Folder.READ_WRITE);
                    browsable.read_only = state.ifolder.getUIDNotSticky();
                    db.folder().setFolderReadOnly(browsable.id, browsable.read_only);
                } catch (ReadOnlyFolderException ex) {
                    state.ifolder.open(Folder.READ_ONLY);
                    browsable.read_only = true;
                    db.folder().setFolderReadOnly(browsable.id, browsable.read_only);
                }

                db.folder().setFolderError(browsable.id, null);

                int count = MessageHelper.getMessageCount(state.ifolder);
                db.folder().setFolderTotal(browsable.id, count < 0 ? null : count, new Date().getTime());

                if (criteria == null) {
                    boolean filter_seen = prefs.getBoolean(FragmentMessages.getFilter(context, "seen", viewType, browsable.type), false);
                    boolean filter_unflagged = prefs.getBoolean(FragmentMessages.getFilter(context, "unflagged", viewType, browsable.type), false);
                    EntityLog.log(context, "Boundary filter seen=" + filter_seen + " unflagged=" + filter_unflagged);

                    List<SearchTerm> and = new ArrayList<>();

                    if (filter_seen &&
                            state.ifolder.getPermanentFlags().contains(Flags.Flag.SEEN))
                        and.add(new FlagTerm(new Flags(Flags.Flag.SEEN), false));

                    if (filter_unflagged &&
                            state.ifolder.getPermanentFlags().contains(Flags.Flag.FLAGGED))
                        and.add(new FlagTerm(new Flags(Flags.Flag.FLAGGED), true));

                    if (and.size() == 0)
                        state.imessages = state.ifolder.getMessages();
                    else
                        state.imessages = state.ifolder.search(new AndTerm(and.toArray(new SearchTerm[0])));

                    EntityLog.log(context, "Boundary filter messages=" + state.imessages.length);
                } else
                    try {
                        Object result = state.ifolder.doCommand(new IMAPFolder.ProtocolCommand() {
                            @Override
                            public Object doCommand(IMAPProtocol protocol) throws ProtocolException {
                                try {
                                    // https://tools.ietf.org/html/rfc3501#section-6.4.4
                                    if (criteria.query != null &&
                                            criteria.query.startsWith("raw:") &&
                                            protocol.hasCapability("X-GM-EXT-1") &&
                                            EntityFolder.ARCHIVE.equals(browsable.type)) {
                                        // https://support.google.com/mail/answer/7190
                                        // https://developers.google.com/gmail/imap/imap-extensions#extension_of_the_search_command_x-gm-raw
                                        Log.i("Boundary raw search=" + criteria.query);

                                        Argument arg = new Argument();
                                        arg.writeAtom("X-GM-RAW");
                                        arg.writeString(criteria.query.substring(4));

                                        Response[] responses = protocol.command("SEARCH", arg);
                                        if (responses.length == 0)
                                            throw new ProtocolException("No response");
                                        if (!responses[responses.length - 1].isOK())
                                            throw new ProtocolException(
                                                    context.getString(R.string.title_service_auth, responses[responses.length - 1]));

                                        List<Integer> msgnums = new ArrayList<>();
                                        for (Response response : responses)
                                            if (((IMAPResponse) response).keyEquals("SEARCH")) {
                                                int msgnum;
                                                while ((msgnum = response.readNumber()) != -1)
                                                    msgnums.add(msgnum);
                                            }

                                        Message[] imessages = new Message[msgnums.size()];
                                        for (int i = 0; i < msgnums.size(); i++)
                                            imessages[i] = state.ifolder.getMessage(msgnums.get(i));

                                        return imessages;
                                    } else {
                                        EntityLog.log(context, "Boundary server" +
                                                " account=" + account +
                                                " folder=" + folder +
                                                " search=" + criteria);

                                        try {
                                            return search(true, browsable.keywords, protocol, state);
                                        } catch (Throwable ex) {
                                            EntityLog.log(context, ex.toString());
                                            if (ex instanceof ProtocolException && ex.getMessage() != null)
                                                try {
                                                    boolean retry = false;
                                                    String remark = null;
                                                    if (ex.getMessage().contains("full text search not supported")) {
                                                        retry = true;
                                                        criteria.in_message = false;
                                                    } else if (ex.getMessage().contains("invalid search criteria")) {
                                                        // invalid SEARCH command syntax, invalid search criteria syntax
                                                        retry = true;
                                                        criteria.in_keywords = false;
                                                        remark = "Keyword search not supported?";
                                                    }

                                                    if (retry) {
                                                        String msg = context.getString(R.string.title_service_auth,
                                                                account.host + ": " +
                                                                        (remark == null ? "" : remark + " - ") +
                                                                        getMessage(ex));
                                                        ApplicationEx.getMainHandler().post(new Runnable() {
                                                            @Override
                                                            public void run() {
                                                                if (intf != null)
                                                                    intf.onWarning(msg);
                                                            }
                                                        });

                                                        return search(true, browsable.keywords, protocol, state);
                                                    }
                                                } catch (Throwable exex) {
                                                    Log.w(exex);
                                                }
                                        }

                                        return search(false, browsable.keywords, protocol, state);
                                    }
                                } catch (Throwable ex) {
                                    ProtocolException pex;
                                    if (ex instanceof ProtocolException)
                                        pex = new ProtocolException(
                                                context.getString(R.string.title_service_auth,
                                                        account.host + ": " + getMessage(ex)),
                                                ex.getCause());
                                    else
                                        pex = new ProtocolException("Search " + account.host, ex);
                                    Log.e(pex);
                                    throw pex;
                                }
                            }
                        });

                        state.imessages = (Message[]) result;
                    } catch (MessagingException ex) {
                        if (ex.getCause() instanceof ProtocolException)
                            throw (ProtocolException) ex.getCause();
                        else
                            throw ex;
                    }
                EntityLog.log(context, "Boundary found messages=" + state.imessages.length);

                FetchProfile fp = new FetchProfile();
                fp.add(UIDFolder.FetchProfileItem.UID);
                fp.add(IMAPFolder.FetchProfileItem.INTERNALDATE);
                state.ifolder.fetch(state.imessages, fp);
                Arrays.sort(state.imessages, new Comparator<Message>() {
                    @Override
                    public int compare(Message m1, Message m2) {
                        Date d1 = null;
                        try {
                            d1 = m1.getReceivedDate();
                        } catch (Throwable ex) {
                            Log.w(ex);
                        }

                        Date d2 = null;
                        try {
                            d2 = m2.getReceivedDate();
                        } catch (Throwable ex) {
                            Log.w(ex);
                        }

                        return Long.compare(d1 == null ? 0 : d1.getTime(), d2 == null ? 0 : d2.getTime());
                    }
                });

                EntityLog.log(context, "Boundary sorted messages=" + state.imessages.length);

                state.index = state.imessages.length - 1;
            } catch (Throwable ex) {
                state.error = true;
                if (ex instanceof FolderClosedException)
                    Log.w("Search", ex);
                else
                    Log.e("Search", ex);
                throw ex;
            }

        List<EntityRule> rules = db.rule().getEnabledRules(browsable.id, false);

        int found = 0;
        while (state.index >= 0 && found < pageSize && !state.destroyed) {
            Log.i("Boundary server index=" + state.index);
            int from = Math.max(0, state.index - (pageSize - found) + 1);
            Message[] isub = Arrays.copyOfRange(state.imessages, from, state.index + 1);
            Arrays.fill(state.imessages, from, state.index + 1, null);
            state.index -= (pageSize - found);

            List<Message> add = new ArrayList<>();
            for (Message m : isub)
                try {
                    long uid = state.ifolder.getUID(m);
                    EntityMessage message = db.message().getMessageByUid(browsable.id, uid);
                    if (message == null)
                        add.add(m);
                } catch (FolderClosedException ex) {
                    throw ex;
                } catch (Throwable ex) {
                    Log.w(ex);
                    add.add(m);
                }

            Log.i("Boundary fetching " + add.size() + "/" + isub.length);
            if (add.size() > 0) {
                FetchProfile fp = new FetchProfile();
                fp.add(FetchProfile.Item.ENVELOPE);
                fp.add(FetchProfile.Item.FLAGS);
                fp.add(FetchProfile.Item.CONTENT_INFO); // body structure
                fp.add(UIDFolder.FetchProfileItem.UID);
                fp.add(IMAPFolder.FetchProfileItem.HEADERS);
                //fp.add(IMAPFolder.FetchProfileItem.MESSAGE);
                fp.add(FetchProfile.Item.SIZE);
                //fp.add(IMAPFolder.FetchProfileItem.INTERNALDATE);
                if (account.isGmail()) {
                    fp.add(GmailFolder.FetchProfileItem.THRID);
                    fp.add(GmailFolder.FetchProfileItem.LABELS);
                }
                state.ifolder.fetch(add.toArray(new Message[0]), fp);
            }

            Core.State astate = new Core.State(ConnectionHelper.getNetworkState(context));
            for (int j = isub.length - 1; j >= 0 && found < pageSize && !state.destroyed && astate.isRecoverable(); j--)
                try {
                    long uid = state.ifolder.getUID(isub[j]);
                    Log.i("Boundary server sync uid=" + uid);
                    EntityMessage message = db.message().getMessageByUid(browsable.id, uid);
                    if (message == null) {
                        message = Core.synchronizeMessage(context,
                                account, browsable,
                                (IMAPStore) state.iservice.getStore(), state.ifolder, (MimeMessage) isub[j],
                                true, true,
                                rules, astate, null);
                        // SQLiteConstraintException
                        if (message != null && criteria == null)
                            found++; // browsed
                    }
                    if (message != null && criteria != null)
                        found += db.message().setMessageFound(message.id, true);
                    Log.i("Boundary matched=" + (message == null ? null : message.id) + " found=" + found);
                } catch (MessageRemovedException | MessageRemovedIOException ex) {
                    Log.w(browsable.name + " boundary server", ex);
                } catch (FolderClosedException ex) {
                    throw ex;
                } catch (IOException ex) {
                    if (ex.getCause() instanceof MessagingException) {
                        Log.w(browsable.name + " boundary server", ex);
                        db.folder().setFolderError(browsable.id, Log.formatThrowable(ex));
                    } else
                        throw ex;
                } catch (Throwable ex) {
                    Log.e(browsable.name + " boundary server", ex);
                    db.folder().setFolderError(browsable.id, Log.formatThrowable(ex));
                } finally {
                    isub[j] = null;
                }
        }

        if (state.index < 0) {
            Log.i("Boundary server end");
            close(state, false);
        }

        Log.i("Boundary server done memory=" + Log.getFreeMemMb());
        return found;
    }

    private String getMessage(Throwable ex) {
        if (ex instanceof ProtocolException) {
            Response r = ((ProtocolException) ex).getResponse();
            if (r != null && !TextUtils.isEmpty(r.getRest()))
                return r.getRest();
        }

        return new ThrowableWrapper(ex).toSafeString();
    }

    private Message[] search(boolean utf8, String[] keywords, IMAPProtocol protocol, State state) throws IOException, MessagingException, ProtocolException {
        EntityLog.log(context, "Search utf8=" + utf8);

        SearchTerm terms = criteria.getTerms(utf8, state.ifolder.getPermanentFlags(), keywords);
        if (terms == null)
            return state.ifolder.getMessages();

        SearchSequence ss = new SearchSequence(protocol);
        Argument args = ss.generateSequence(terms, utf8 ? StandardCharsets.UTF_8.name() : null);
        args.writeAtom("ALL");

        // https://www.rfc-editor.org/rfc/rfc3501#section-6.4.4
        Response[] responses = protocol.command("SEARCH" + (utf8 ? " CHARSET UTF-8" : ""), args);
        if (responses != null && responses.length > 0 && !responses[responses.length - 1].isOK()) {
            // Normally "NO", but some servers respond with "BAD Could not parse command"
            if (!BuildConfig.PLAY_STORE_RELEASE)
                Log.e(responses[responses.length - 1].toString());
            responses = protocol.command("SEARCH", args);
        }
        if (responses == null || responses.length == 0)
            throw new ProtocolException("No response from server");
        for (Response response : responses)
            Log.i("Search response=" + response);
        if (!responses[responses.length - 1].isOK())
            throw new ProtocolException(responses[responses.length - 1]);

        List<Integer> msgnums = new ArrayList<>();
        for (int r = 0; r < responses.length; r++) {
            IMAPResponse response = (IMAPResponse) responses[r];
            if (response.keyEquals("SEARCH")) {
                int msgnum;
                while ((msgnum = response.readNumber()) != -1)
                    msgnums.add(msgnum);
            }
        }

        EntityLog.log(context, "Search messages=" + msgnums.size());
        Message[] imessages = new Message[msgnums.size()];
        for (int i = 0; i < msgnums.size(); i++)
            imessages[i] = state.ifolder.getMessage(msgnums.get(i));

        return imessages;
    }

    private static boolean matchMessage(Context context, EntityMessage message, SearchCriteria criteria, boolean partial) {
        if (criteria.with_unseen) {
            if (message.ui_seen)
                return false;
        }

        if (criteria.with_flagged) {
            if (!message.ui_flagged)
                return false;
        }

        if (criteria.with_hidden) {
            if (message.ui_snoozed == null)
                return false;
        }

        if (criteria.with_encrypted) {
            if (message.encrypt == null ||
                    EntityMessage.ENCRYPT_NONE.equals(message.encrypt))
                return false;
        }

        if (criteria.with_attachments) {
            if (message.attachments == 0)
                return false;
        }

        //
        if (criteria.with_notes) {
            if (message.notes == null)
                return false;
        }

        //
        if (criteria.with_size != null) {
            if (message.total == null || message.total < criteria.with_size)
                return false;
        }

        //
        if (criteria.before != null) {
            if (message.received > criteria.before)
                return false;
        }

        //
        if (criteria.after != null) {
            if (message.received < criteria.after)
                return false;
        }

        if (criteria.in_senders) {
            if (contains(message.from, criteria.query, partial))
                return true;
        }

        if (criteria.in_recipients) {
            if (contains(message.to, criteria.query, partial) ||
                    contains(message.cc, criteria.query, partial) ||
                    contains(message.bcc, criteria.query, partial))
                return true;
        }

        if (criteria.in_subject) {
            if (contains(message.subject, criteria.query, partial, false))
                return true;
        }

        if (criteria.in_keywords) {
            if (message.keywords != null)
                for (String keyword : message.keywords)
                    if (contains(keyword, criteria.query, partial, false))
                        return true;
        }

        if (criteria.in_notes) {
            if (contains(message.notes, criteria.query, partial, false))
                return true;
        }

        if (criteria.in_filenames) {
            DB db = DB.getInstance(context);
            List<EntityAttachment> attachments = db.attachment().getAttachments(message.id);
            if (attachments != null)
                for (EntityAttachment attachment : attachments)
                    if (!TextUtils.isEmpty(attachment.name) && contains(attachment.name, criteria.query, true, false))
                        return true; // Partial search to find "filename.extension"
        }

        if (criteria.in_headers) {
            if (message.headers != null && message.headers.contains(criteria.query))
                return true;
        }

        if (criteria.in_html || criteria.in_message)
            try {
                File file = EntityMessage.getFile(context, message.id);
                if (file.exists()) {
                    String selector = criteria.getJsoup();
                    if (selector != null) {
                        Document d = JsoupEx.parse(file);
                        return (d.select(selector).size() > 0);
                    }

                    String html = Helper.readText(file);
                    if (criteria.in_html) {
                        if (html.contains(criteria.query))
                            return true;
                    }

                    if (criteria.in_message) {
                        // This won't match <p>An <b>example</b><p> when searching for "An example"
                        if (contains(html, criteria.query, partial, true)) {
                            String text = HtmlHelper.getFullText(html);
                            if (contains(text, criteria.query, partial, false))
                                return true;
                        }
                    }
                }
            } catch (IOException ex) {
                Log.e(ex);
            }

        return false;
    }

    private static boolean contains(Address[] addresses, String query, boolean partial) {
        if (addresses == null)
            return false;
        for (Address address : addresses)
            if (contains(address.toString(), query, partial, false))
                return true;
        return false;
    }

    private static boolean contains(String text, String query, boolean partial, boolean html) {
        if (TextUtils.isEmpty(text))
            return false;

        text = Fts4DbHelper.processBreakText(text);

        List<String> word = new ArrayList<>();
        for (String w : query.trim().split("\\s+"))
            if (w.length() > 1 && w.startsWith("+")) {
                if (!text.contains(Fts4DbHelper.preprocessText(w.substring(1))))
                    return false;
            } else if (w.length() > 1 && w.startsWith("-")) {
                if (!html && text.contains(Fts4DbHelper.preprocessText(w.substring(1))))
                    return false;
            } else
                word.addAll(Arrays.asList(Fts4DbHelper.processBreakText(w).split("\\s+")));

        if (word.size() == 0)
            return true;

        // \b is limited to [0-9A-Za-z_]
        String b = "(^|\\s+)";
        String a = "($|\\s+)";

        StringBuilder sb = new StringBuilder();
        sb.append(partial ? ".*(" : ".*?" + b + "(");
        for (int i = 0; i < word.size(); i++) {
            if (i > 0)
                sb.append("\\s+");
            sb.append(Pattern.quote(word.get(i)));
        }
        sb.append(partial ? ").*" : ")" + a + ".*?");

        Pattern pat = Pattern.compile(sb.toString(), Pattern.DOTALL);
        return pat.matcher(text).matches();
    }

    State getState() {
        return this.state;
    }

    void destroy(State state) {
        state.destroyed = true;
        this.intf = null;
        Log.i("Boundary destroy");

        executor.submit(new Runnable() {
            @Override
            public void run() {
                close(state, true);
            }
        });
    }

    private void close(State state, boolean reset) {
        Log.i("Boundary close");

        try {
            if (state.ifolder != null && state.ifolder.isOpen())
                state.ifolder.close(false);
        } catch (Throwable ex) {
            Log.e("Boundary", ex);
        }

        try {
            if (state.iservice != null && state.iservice.isOpen())
                state.iservice.close();
        } catch (Throwable ex) {
            Log.e("Boundary", ex);
        }

        if (reset)
            state.reset();
    }

    static class State {
        final AtomicInteger queued = new AtomicInteger(0);
        boolean destroyed = false;
        boolean error = false;
        int index = 0;
        int offset = 0;
        List<Long> ids = null;
        List<TupleMatch> matches = null;

        EmailService iservice = null;
        IMAPFolder ifolder = null;
        Message[] imessages = null;

        void reset() {
            Log.i("Boundary reset");
            queued.set(0);
            destroyed = false;
            error = false;
            index = 0;
            offset = 0;
            ids = null;
            matches = null;
            iservice = null;
            ifolder = null;
            imessages = null;

            Helper.gc();
        }
    }

    static class SearchCriteria extends EntitySearch implements Serializable {
        String query;
        boolean fts = false;
        boolean in_senders = true;
        boolean in_recipients = true;
        boolean in_subject = true;
        boolean in_keywords = true;
        boolean in_message = true;
        boolean in_notes = true;
        boolean in_filenames = false;
        boolean in_headers = false;
        boolean in_html = false;
        boolean with_unseen;
        boolean with_flagged;
        boolean with_hidden;
        boolean with_encrypted;
        boolean with_attachments;
        boolean with_notes;
        String[] with_types;
        Integer with_size = null;
        boolean in_trash = true;
        boolean in_junk = true;
        Long after = null;
        Long before = null;

        private static final String FROM = "from:";
        private static final String TO = "to:";
        private static final String CC = "cc:";
        private static final String BCC = "bcc:";
        private static final String KEYWORD = "keyword:";
        private static final String JSOUP_PREFIX = "jsoup:";

        String getJsoup() {
            if (query == null)
                return null;
            if (!query.startsWith(JSOUP_PREFIX))
                return null;
            return query.substring(JSOUP_PREFIX.length());
        }

        boolean onServer() {
            if (query == null)
                return false;

            for (String w : query.trim().split("\\s+"))
                if (w.length() > 1 && w.startsWith("?"))
                    return true;
                else if (w.length() > FROM.length() && w.startsWith(FROM))
                    return true;
                else if (w.length() > TO.length() && w.startsWith(TO))
                    return true;
                else if (w.length() > CC.length() && w.startsWith(CC))
                    return true;
                else if (w.length() > BCC.length() && w.startsWith(BCC))
                    return true;
                else if (w.length() > KEYWORD.length() && w.startsWith(KEYWORD))
                    return true;

            return false;
        }

        SearchTerm getTerms(boolean utf8, Flags flags, String[] keywords) {
            List<SearchTerm> or = new ArrayList<>();
            List<SearchTerm> and = new ArrayList<>();
            if (query != null) {
                String search = query;

                if (!utf8) {
                    // Perhaps:  Transliterator.getInstance("de-ASCII");
                    search = search
                            .replace("ß", "ss") // Eszett
                            .replace("ij", "ij")
                            .replace("ø", "o");
                    search = Normalizer
                            .normalize(search, Normalizer.Form.NFKD)
                            .replaceAll("[^\\p{ASCII}]", "");
                    if (TextUtils.isEmpty(search)) {
                        String msg = "Cannot convert to ASCII: " + query;
                        Log.e(msg);
                        throw new IllegalArgumentException(msg);
                    }
                }

                List<String> word = new ArrayList<>();
                List<String> plus = new ArrayList<>();
                List<String> minus = new ArrayList<>();
                List<String> opt = new ArrayList<>();
                List<String> andFrom = new ArrayList<>();
                List<String> andTo = new ArrayList<>();
                List<String> andCc = new ArrayList<>();
                List<String> andBcc = new ArrayList<>();
                List<String> andKeyword = new ArrayList<>();
                StringBuilder all = new StringBuilder();
                for (String w : search.trim().split("\\s+")) {
                    if (all.length() > 0)
                        all.append(' ');

                    if (w.length() > 1 && w.startsWith("+")) {
                        plus.add(w.substring(1));
                        all.append(w.substring(1));
                    } else if (w.length() > 1 && w.startsWith("-")) {
                        minus.add(w.substring(1));
                        all.append(w.substring(1));
                    } else if (w.length() > 1 && w.startsWith("?")) {
                        opt.add(w.substring(1));
                        all.append(w.substring(1));
                    } else if (w.length() > FROM.length() && w.startsWith(FROM))
                        andFrom.add(w.substring(FROM.length()));
                    else if (w.length() > TO.length() && w.startsWith(TO))
                        andTo.add(w.substring(TO.length()));
                    else if (w.length() > CC.length() && w.startsWith(CC))
                        andCc.add(w.substring(CC.length()));
                    else if (w.length() > BCC.length() && w.startsWith(BCC))
                        andBcc.add(w.substring(BCC.length()));
                    else if (w.length() > KEYWORD.length() && w.startsWith(KEYWORD))
                        andKeyword.add(w.substring(KEYWORD.length()));
                    else {
                        word.add(w);
                        all.append(w);
                    }
                }

                if (plus.size() + minus.size() + opt.size() +
                        andFrom.size() + andTo.size() + andCc.size() + andBcc.size() + andKeyword.size() > 0)
                    search = all.toString();

                // Yahoo! does not support keyword search, but uses the flags $Forwarded $Junk $NotJunk
                boolean hasKeywords = false;
                for (String keyword : keywords)
                    if (!keyword.startsWith("$")) {
                        hasKeywords = true;
                        break;
                    }

                if (andFrom.size() > 0) {
                    for (String term : andFrom)
                        and.add(new FromStringTerm(term));
                } else {
                    if (in_senders && !TextUtils.isEmpty(search) &&
                            plus.size() + minus.size() + opt.size() == 0)
                        or.add(new FromStringTerm(search));
                }

                if (andTo.size() + andCc.size() + andBcc.size() > 0) {
                    for (String term : andTo)
                        and.add(new RecipientStringTerm(Message.RecipientType.TO, term));
                    for (String term : andCc)
                        and.add(new RecipientStringTerm(Message.RecipientType.CC, term));
                    for (String term : andBcc)
                        and.add(new RecipientStringTerm(Message.RecipientType.BCC, term));
                } else {
                    if (in_recipients && !TextUtils.isEmpty(search) &&
                            plus.size() + minus.size() + opt.size() == 0) {
                        or.add(new RecipientStringTerm(Message.RecipientType.TO, search));
                        or.add(new RecipientStringTerm(Message.RecipientType.CC, search));
                        or.add(new RecipientStringTerm(Message.RecipientType.BCC, search));
                    }
                }

                if (in_subject && !TextUtils.isEmpty(search))
                    if (plus.size() + minus.size() + opt.size() == 0)
                        or.add(new SubjectTerm(search));
                    else
                        try {
                            or.add(construct(word, plus, minus, opt, SubjectTerm.class));
                        } catch (Throwable ex) {
                            Log.e(ex);
                            or.add(new SubjectTerm(search));
                        }

                if (hasKeywords)
                    if (andKeyword.size() > 0) {
                        for (String term : andKeyword)
                            and.add(new FlagTerm(new Flags(term), true));
                    } else {
                        if (in_keywords && !TextUtils.isEmpty(search) &&
                                plus.size() + minus.size() + opt.size() == 0) {
                            String keyword = MessageHelper.sanitizeKeyword(search);
                            if (TextUtils.isEmpty(keyword))
                                Log.w("Keyword empty=" + search);
                            else
                                or.add(new FlagTerm(new Flags(keyword), true));
                        }
                    }

                if (in_message && !TextUtils.isEmpty(search))
                    if (plus.size() + minus.size() + opt.size() == 0)
                        or.add(new BodyTerm(search));
                    else
                        try {
                            or.add(construct(word, plus, minus, opt, BodyTerm.class));
                        } catch (Throwable ex) {
                            Log.e(ex);
                            or.add(new BodyTerm(search));
                        }
            }

            if (with_unseen && flags.contains(Flags.Flag.SEEN))
                and.add(new FlagTerm(new Flags(Flags.Flag.SEEN), false));
            if (with_flagged && flags.contains(Flags.Flag.FLAGGED))
                and.add(new FlagTerm(new Flags(Flags.Flag.FLAGGED), true));

            if (with_size != null)
                and.add(new SizeTerm(ComparisonTerm.GT, with_size));

            if (after != null)
                and.add(new ReceivedDateTerm(ComparisonTerm.GE, new Date(after)));
            if (before != null)
                and.add(new ReceivedDateTerm(ComparisonTerm.LE, new Date(before)));

            SearchTerm term = null;

            if (or.size() > 0)
                term = new OrTerm(or.toArray(new SearchTerm[0]));

            if (and.size() > 0)
                if (term == null)
                    term = new AndTerm(and.toArray(new SearchTerm[0]));
                else
                    term = new AndTerm(term, new AndTerm(and.toArray(new SearchTerm[0])));

            return term;
        }

        private SearchTerm construct(
                List<String> word,
                List<String> plus,
                List<String> minus,
                List<String> opt,
                Class<?> clazz) throws ReflectiveOperationException {
            SearchTerm term = null;
            Constructor<?> ctor = clazz.getConstructor(String.class);

            if (word.size() > 0)
                term = (SearchTerm) ctor.newInstance(TextUtils.join(" ", word));

            for (String p : plus)
                if (term == null)
                    term = (SearchTerm) ctor.newInstance(p);
                else
                    term = new AndTerm(term, (SearchTerm) ctor.newInstance(p));

            for (String m : minus)
                if (term == null)
                    term = new NotTerm((SearchTerm) ctor.newInstance(m));
                else
                    term = new AndTerm(term, new NotTerm((SearchTerm) ctor.newInstance(m)));

            for (String o : opt)
                if (term == null)
                    term = (SearchTerm) ctor.newInstance(o);
                else
                    term = new OrTerm(term, (SearchTerm) ctor.newInstance(o));

            return term;
        }

        String getTitle(Context context) {
            List<String> flags = new ArrayList<>();
            if (with_unseen)
                flags.add(context.getString(R.string.title_search_flag_unseen));
            if (with_flagged)
                flags.add(context.getString(R.string.title_search_flag_flagged));
            if (with_hidden)
                flags.add(context.getString(R.string.title_search_flag_hidden));
            if (with_encrypted)
                flags.add(context.getString(R.string.title_search_flag_encrypted));
            if (with_attachments)
                flags.add(context.getString(R.string.title_search_flag_attachments));
            if (with_notes)
                flags.add(context.getString(R.string.title_search_flag_notes));
            if (with_types != null)
                if (with_types.length == 1 && "text/calendar".equals(with_types[0]))
                    flags.add(context.getString(R.string.title_search_flag_invite));
                else
                    flags.add(TextUtils.join(", ", with_types));
            if (with_size != null)
                flags.add(context.getString(R.string.title_search_flag_size,
                        Helper.humanReadableByteCount(with_size)));
            return (query == null ? "" : query + " ")
                    + (flags.size() > 0 ? "+" : "")
                    + TextUtils.join(",", flags);
        }

        @Override
        public boolean equals(@Nullable Object obj) {
            if (obj instanceof SearchCriteria) {
                SearchCriteria other = (SearchCriteria) obj;
                return (Objects.equals(this.query, other.query) &&
                        this.fts == other.fts &&
                        this.in_senders == other.in_senders &&
                        this.in_recipients == other.in_recipients &&
                        this.in_subject == other.in_subject &&
                        this.in_keywords == other.in_keywords &&
                        this.in_message == other.in_message &&
                        this.in_notes == other.in_notes &&
                        this.in_filenames == other.in_filenames &&
                        this.in_headers == other.in_headers &&
                        this.in_html == other.in_html &&
                        this.with_unseen == other.with_unseen &&
                        this.with_flagged == other.with_flagged &&
                        this.with_hidden == other.with_hidden &&
                        this.with_encrypted == other.with_encrypted &&
                        this.with_attachments == other.with_attachments &&
                        this.with_notes == other.with_notes &&
                        Arrays.equals(this.with_types, other.with_types) &&
                        Objects.equals(this.with_size, other.with_size) &&
                        this.in_trash == other.in_trash &&
                        this.in_junk == other.in_junk &&
                        Objects.equals(this.after, other.after) &&
                        Objects.equals(this.before, other.before));
            } else
                return false;
        }

        JSONObject toJsonData() throws JSONException {
            JSONObject json = new JSONObject();
            json.put("query", query);
            json.put("fts", fts);
            json.put("in_senders", in_senders);
            json.put("in_recipients", in_recipients);
            json.put("in_subject", in_subject);
            json.put("in_keywords", in_keywords);
            json.put("in_message", in_message);
            json.put("in_notes", in_notes);
            json.put("in_filenames", in_filenames);
            json.put("in_headers", in_headers);
            json.put("in_html", in_html);
            json.put("with_unseen", with_unseen);
            json.put("with_flagged", with_flagged);
            json.put("with_hidden", with_hidden);
            json.put("with_encrypted", with_encrypted);
            json.put("with_attachments", with_attachments);
            json.put("with_notes", with_notes);

            if (with_types != null) {
                JSONArray jtypes = new JSONArray();
                for (String type : with_types)
                    jtypes.put(type);
                json.put("with_types", jtypes);
            }

            if (with_size != null)
                json.put("with_size", with_size);

            json.put("in_trash", in_trash);
            json.put("in_junk", in_junk);

            Calendar now = Calendar.getInstance();
            now.set(Calendar.MILLISECOND, 0);
            now.set(Calendar.SECOND, 0);
            now.set(Calendar.MINUTE, 0);
            now.set(Calendar.HOUR_OF_DAY, 0);

            if (after != null)
                json.put("after", after - now.getTimeInMillis());

            if (before != null)
                json.put("before", before - now.getTimeInMillis());

            return json;
        }

        public static SearchCriteria fromJsonData(JSONObject json) throws JSONException {
            SearchCriteria criteria = new SearchCriteria();
            criteria.query = json.optString("query");
            criteria.fts = json.optBoolean("fts");
            criteria.in_senders = json.optBoolean("in_senders");
            criteria.in_recipients = json.optBoolean("in_recipients");
            criteria.in_subject = json.optBoolean("in_subject");
            criteria.in_keywords = json.optBoolean("in_keywords");
            criteria.in_message = json.optBoolean("in_message");
            criteria.in_notes = json.optBoolean("in_notes");
            criteria.in_filenames = json.optBoolean("in_filenames");
            criteria.in_headers = json.optBoolean("in_headers");
            criteria.in_html = json.optBoolean("in_html");
            criteria.with_unseen = json.optBoolean("with_unseen");
            criteria.with_flagged = json.optBoolean("with_flagged");
            criteria.with_hidden = json.optBoolean("with_hidden");
            criteria.with_encrypted = json.optBoolean("with_encrypted");
            criteria.with_attachments = json.optBoolean("with_attachments");
            criteria.with_notes = json.optBoolean("with_notes");

            if (json.has("with_types")) {
                JSONArray jtypes = json.getJSONArray("with_types");
                criteria.with_types = new String[jtypes.length()];
                for (int i = 0; i < jtypes.length(); i++)
                    criteria.with_types[i] = jtypes.getString(i);
            }

            if (json.has("with_size"))
                criteria.with_size = json.getInt("with_size");

            criteria.in_trash = json.optBoolean("in_trash");
            criteria.in_junk = json.optBoolean("in_junk");

            Calendar now = Calendar.getInstance();
            now.set(Calendar.MILLISECOND, 0);
            now.set(Calendar.SECOND, 0);
            now.set(Calendar.MINUTE, 0);
            now.set(Calendar.HOUR_OF_DAY, 0);

            if (json.has("after"))
                criteria.after = json.getLong("after") + now.getTimeInMillis();

            if (json.has("before"))
                criteria.before = json.getLong("before") + now.getTimeInMillis();

            return criteria;
        }

        @NonNull
        @Override
        public String toString() {
            return query +
                    " fts=" + fts +
                    " senders=" + in_senders +
                    " recipients=" + in_recipients +
                    " subject=" + in_subject +
                    " keywords=" + in_keywords +
                    " message=" + in_message +
                    " notes=" + in_notes +
                    " filenames=" + in_filenames +
                    " headers=" + in_headers +
                    " html=" + in_html +
                    " unseen=" + with_unseen +
                    " flagged=" + with_flagged +
                    " hidden=" + with_hidden +
                    " encrypted=" + with_encrypted +
                    " w/attachments=" + with_attachments +
                    " w/notes=" + with_notes +
                    " type=" + (with_types == null ? null : TextUtils.join(",", with_types)) +
                    " size=" + with_size +
                    " trash=" + in_trash +
                    " junk=" + in_junk +
                    " after=" + (after == null ? "" : new Date(after)) +
                    " before=" + (before == null ? "" : new Date(before));
        }
    }
}