diff --git a/app/src/main/java/eu/faircode/email/Core.java b/app/src/main/java/eu/faircode/email/Core.java
index 7d5c244c91..5c9d6e87b6 100644
--- a/app/src/main/java/eu/faircode/email/Core.java
+++ b/app/src/main/java/eu/faircode/email/Core.java
@@ -662,13 +662,10 @@ class Core {
             message = db.message().getMessage(message.id);
 
             List<String> keywords = new ArrayList<>(Arrays.asList(message.keywords));
-            if (set) {
-                if (!keywords.contains(keyword))
-                    keywords.add(keyword);
-            } else {
-                while (keywords.remove(keyword))
-                    ;
-            }
+            while (keywords.remove(keyword))
+                ;
+            if (set)
+                keywords.add(keyword);
 
             db.message().setMessageKeywords(message.id, DB.Converters.fromStringArray(keywords.toArray(new String[0])));