diff --git a/app/src/main/java/eu/faircode/email/AdapterMessage.java b/app/src/main/java/eu/faircode/email/AdapterMessage.java
index 1e4154067e..0d99cb7ae9 100644
--- a/app/src/main/java/eu/faircode/email/AdapterMessage.java
+++ b/app/src/main/java/eu/faircode/email/AdapterMessage.java
@@ -1128,7 +1128,7 @@ public class AdapterMessage extends RecyclerView.Adapter<AdapterMessage.ViewHold
             if (show_recipients && recipients != null)
                 all.addAll(Arrays.asList(recipients));
             ContactInfo[] info = ContactInfo.getCached(context, message.account, message.folderType, all.toArray(new Address[0]));
-            if (info == null) {
+            if (info == null && avatars) {
                 if (taskContactInfo != null)
                     taskContactInfo.cancel(context);