diff --git a/apps/web/app/api/google/webhook/process-history-item.test.ts b/apps/web/app/api/google/webhook/process-history-item.test.ts
index 97f812e70c..a1ca0c41f4 100644
--- a/apps/web/app/api/google/webhook/process-history-item.test.ts
+++ b/apps/web/app/api/google/webhook/process-history-item.test.ts
@@ -3,7 +3,8 @@ import {
   shouldRunColdEmailBlocker,
   processHistoryItem,
 } from "./process-history-item";
-import { ColdEmailSetting } from "@prisma/client";
+import { HistoryEventType } from "./types";
+import { ColdEmailSetting, ColdEmailStatus } from "@prisma/client";
 import type { gmail_v1 } from "@googleapis/gmail";
 import { isAssistantEmail } from "@/utils/assistant/is-assistant-email";
 import { runColdEmailBlockerWithProvider } from "@/utils/cold-email/is-cold-email";
@@ -16,6 +17,10 @@ import { runRules } from "@/utils/ai/choose-rule/run-rules";
 import { processAssistantEmail } from "@/utils/assistant/process-assistant-email";
 import { getEmailAccount } from "@/__tests__/helpers";
 import { enqueueDigestItem } from "@/utils/digest/index";
+import prisma from "@/utils/__mocks__/prisma";
+import { saveLearnedPatterns } from "@/utils/rule/learned-patterns";
+import { createEmailProvider } from "@/utils/email/provider";
+import { inboxZeroLabels } from "@/utils/label";
 
 vi.mock("server-only", () => ({}));
 vi.mock("next/server", () => ({
@@ -91,6 +96,26 @@ vi.mock("@/utils/email/provider", () => ({
   }),
 }));
 
+vi.mock("@/utils/gmail/label", async () => {
+  const actual = await vi.importActual("@/utils/gmail/label");
+  return {
+    ...actual,
+    getLabelById: vi.fn().mockImplementation(async ({ id }: { id: string }) => {
+      const labelMap: Record<string, { name: string }> = {
+        "label-1": { name: inboxZeroLabels.cold_email.name },
+        "label-2": { name: "Newsletter" },
+        "label-3": { name: "Marketing" },
+        "label-4": { name: "To Reply" },
+      };
+      return labelMap[id] || { name: "Unknown Label" };
+    }),
+  };
+});
+
+vi.mock("@/utils/rule/learned-patterns", () => ({
+  saveLearnedPatterns: vi.fn().mockResolvedValue(undefined),
+}));
+
 describe("processHistoryItem", () => {
   beforeEach(() => {
     vi.clearAllMocks();
@@ -99,9 +124,34 @@ describe("processHistoryItem", () => {
   const createHistoryItem = (
     messageId = "123",
     threadId = "thread-123",
-  ): gmail_v1.Schema$HistoryMessageAdded => ({
-    message: { id: messageId, threadId },
-  });
+    type: HistoryEventType = HistoryEventType.MESSAGE_ADDED,
+    labelIds?: string[],
+  ) => {
+    const baseItem = { message: { id: messageId, threadId } };
+
+    if (type === HistoryEventType.LABEL_REMOVED) {
+      return {
+        type,
+        item: {
+          ...baseItem,
+          labelIds: labelIds || [],
+        } as gmail_v1.Schema$HistoryLabelRemoved,
+      };
+    } else if (type === HistoryEventType.LABEL_ADDED) {
+      return {
+        type,
+        item: {
+          ...baseItem,
+          labelIds: labelIds || [],
+        } as gmail_v1.Schema$HistoryLabelAdded,
+      };
+    } else {
+      return {
+        type,
+        item: baseItem as gmail_v1.Schema$HistoryMessageAdded,
+      };
+    }
+  };
 
   const defaultOptions = {
     gmail: {} as any,
@@ -160,7 +210,6 @@ describe("processHistoryItem", () => {
   });
 
   it("should skip if message is outbound", async () => {
-    const { createEmailProvider } = await import("@/utils/email/provider");
     vi.mocked(createEmailProvider).mockResolvedValueOnce({
       getMessage: vi.fn().mockResolvedValue({
         id: "123",
diff --git a/apps/web/app/api/google/webhook/process-history-item.ts b/apps/web/app/api/google/webhook/process-history-item.ts
index c1db10c997..5b513e7dc1 100644
--- a/apps/web/app/api/google/webhook/process-history-item.ts
+++ b/apps/web/app/api/google/webhook/process-history-item.ts
@@ -25,11 +25,17 @@ import type { EmailAccountWithAI } from "@/utils/llms/types";
 import { formatError } from "@/utils/error";
 import { createEmailProvider } from "@/utils/email/provider";
 import { enqueueDigestItem } from "@/utils/digest/index";
+import { HistoryEventType } from "@/app/api/google/webhook/types";
+import { handleLabelRemovedEvent } from "@/app/api/google/webhook/process-label-removed-event";
 
 export async function processHistoryItem(
-  {
-    message,
-  }: gmail_v1.Schema$HistoryMessageAdded | gmail_v1.Schema$HistoryLabelAdded,
+  historyItem: {
+    type: HistoryEventType;
+    item:
+      | gmail_v1.Schema$HistoryMessageAdded
+      | gmail_v1.Schema$HistoryLabelAdded
+      | gmail_v1.Schema$HistoryLabelRemoved;
+  },
   {
     gmail,
     emailAccount,
@@ -39,8 +45,10 @@ export async function processHistoryItem(
     rules,
   }: ProcessHistoryOptions,
 ) {
-  const messageId = message?.id;
-  const threadId = message?.threadId;
+  const { type, item } = historyItem;
+  const messageId = item.message?.id;
+  const threadId = item.message?.threadId;
+
   const emailAccountId = emailAccount.id;
   const userEmail = emailAccount.email;
 
@@ -52,6 +60,23 @@ export async function processHistoryItem(
     threadId,
   };
 
+  const provider = await createEmailProvider({
+    emailAccountId,
+    provider: "google",
+  });
+
+  if (type === HistoryEventType.LABEL_REMOVED) {
+    logger.info("Processing label removed event for learning", loggerOptions);
+    return handleLabelRemovedEvent(item, {
+      gmail,
+      emailAccount,
+      provider,
+    });
+  } else if (type === HistoryEventType.LABEL_ADDED) {
+    logger.info("Processing label added event for learning", loggerOptions);
+    return;
+  }
+
   const isFree = await markMessageAsProcessing({ userEmail, messageId });
 
   if (!isFree) {
@@ -61,14 +86,9 @@ export async function processHistoryItem(
 
   logger.info("Getting message", loggerOptions);
 
-  const emailProvider = await createEmailProvider({
-    emailAccountId,
-    provider: "google",
-  });
-
   try {
     const [parsedMessage, hasExistingRule] = await Promise.all([
-      emailProvider.getMessage(messageId),
+      provider.getMessage(messageId),
       prisma.executedRule.findUnique({
         where: {
           unique_emailAccount_thread_message: {
@@ -97,11 +117,6 @@ export async function processHistoryItem(
       emailToCheck: parsedMessage.headers.to,
     });
 
-    const provider = await createEmailProvider({
-      emailAccountId,
-      provider: "google",
-    });
-
     if (isForAssistant) {
       logger.info("Passing through assistant email.", loggerOptions);
       return processAssistantEmail({
diff --git a/apps/web/app/api/google/webhook/process-history.ts b/apps/web/app/api/google/webhook/process-history.ts
index 250d52ddf4..4848213ee1 100644
--- a/apps/web/app/api/google/webhook/process-history.ts
+++ b/apps/web/app/api/google/webhook/process-history.ts
@@ -8,7 +8,10 @@ import { ColdEmailSetting } from "@prisma/client";
 import { captureException } from "@/utils/error";
 import { unwatchEmails } from "@/app/api/watch/controller";
 import { createEmailProvider } from "@/utils/email/provider";
-import type { ProcessHistoryOptions } from "@/app/api/google/webhook/types";
+import {
+  HistoryEventType,
+  type ProcessHistoryOptions,
+} from "@/app/api/google/webhook/types";
 import { processHistoryItem } from "@/app/api/google/webhook/process-history-item";
 import { logger } from "@/app/api/google/webhook/logger";
 import { getHistory } from "@/utils/gmail/history";
@@ -162,7 +165,7 @@ export async function processHistoryForUser(
       // NOTE this can cause problems if we're way behind
       // NOTE this doesn't include startHistoryId in the results
       startHistoryId,
-      historyTypes: ["messageAdded", "labelAdded"],
+      historyTypes: ["messageAdded", "labelAdded", "labelRemoved"],
       maxResults: 500,
     });
 
@@ -232,26 +235,43 @@ async function processHistory(options: ProcessHistoryOptions) {
     const historyMessages = [
       ...(h.messagesAdded || []),
       ...(h.labelsAdded || []),
+      ...(h.labelsRemoved || []),
     ];
 
     if (!historyMessages.length) continue;
 
-    const inboxMessages = historyMessages.filter(isInboxOrSentMessage);
-    const uniqueMessages = uniqBy(inboxMessages, (m) => m.message?.id);
+    const allEvents = [
+      ...(h.messagesAdded || [])
+        .filter(isInboxOrSentMessage)
+        .map((m) => ({ type: HistoryEventType.MESSAGE_ADDED, item: m })),
+      ...(h.labelsAdded || []).map((m) => ({
+        type: HistoryEventType.LABEL_ADDED,
+        item: m,
+      })),
+      ...(h.labelsRemoved || []).map((m) => ({
+        type: HistoryEventType.LABEL_REMOVED,
+        item: m,
+      })),
+    ];
+
+    const uniqueEvents = uniqBy(
+      allEvents,
+      (e) => `${e.type}:${e.item.message?.id}`,
+    );
 
-    for (const m of uniqueMessages) {
+    for (const event of uniqueEvents) {
       try {
-        await processHistoryItem(m, options);
+        await processHistoryItem(event, options);
       } catch (error) {
         captureException(
           error,
-          { extra: { userEmail, messageId: m.message?.id } },
+          { extra: { userEmail, messageId: event.item.message?.id } },
           userEmail,
         );
         logger.error("Error processing history item", {
           userEmail,
-          messageId: m.message?.id,
-          threadId: m.message?.threadId,
+          messageId: event.item.message?.id,
+          threadId: event.item.message?.threadId,
           error:
             error instanceof Error
               ? {
diff --git a/apps/web/app/api/google/webhook/process-label-removed-event.test.ts b/apps/web/app/api/google/webhook/process-label-removed-event.test.ts
new file mode 100644
index 0000000000..da4aa58214
--- /dev/null
+++ b/apps/web/app/api/google/webhook/process-label-removed-event.test.ts
@@ -0,0 +1,210 @@
+import { vi, describe, it, expect, beforeEach } from "vitest";
+import { ColdEmailStatus } from "@prisma/client";
+import { HistoryEventType } from "./types";
+import { handleLabelRemovedEvent } from "./process-label-removed-event";
+import type { gmail_v1 } from "@googleapis/gmail";
+import { inboxZeroLabels } from "@/utils/label";
+import { saveLearnedPatterns } from "@/utils/rule/learned-patterns";
+import prisma from "@/utils/__mocks__/prisma";
+
+vi.mock("server-only", () => ({}));
+
+// Mock dependencies
+vi.mock("@/utils/prisma");
+vi.mock("@/utils/rule/learned-patterns", () => ({
+  saveLearnedPatterns: vi.fn().mockResolvedValue(undefined),
+}));
+
+vi.mock("@/utils/gmail/label", () => ({
+  GmailLabel: {
+    INBOX: "INBOX",
+    SENT: "SENT",
+    UNREAD: "UNREAD",
+    STARRED: "STARRED",
+    IMPORTANT: "IMPORTANT",
+    SPAM: "SPAM",
+    TRASH: "TRASH",
+    DRAFT: "DRAFT",
+    PERSONAL: "CATEGORY_PERSONAL",
+    SOCIAL: "CATEGORY_SOCIAL",
+    PROMOTIONS: "CATEGORY_PROMOTIONS",
+    FORUMS: "CATEGORY_FORUMS",
+    UPDATES: "CATEGORY_UPDATES",
+  },
+  getLabelById: vi.fn().mockImplementation(({ id }: { id: string }) => {
+    const labelMap: Record<string, { name: string }> = {
+      "label-1": { name: inboxZeroLabels.cold_email.name },
+      "label-2": { name: "Newsletter" },
+      "label-3": { name: "Marketing" },
+      "label-4": { name: "To Reply" },
+    };
+    return Promise.resolve(labelMap[id] || { name: "Unknown Label" });
+  }),
+}));
+
+vi.mock("@/utils/email", () => ({
+  extractEmailAddress: vi.fn().mockReturnValue("sender@example.com"),
+}));
+
+describe("process-label-removed-event", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  const createLabelRemovedHistoryItem = (
+    messageId = "123",
+    threadId = "thread-123",
+    labelIds = ["label-1"],
+  ) => ({
+    type: HistoryEventType.LABEL_REMOVED,
+    item: {
+      message: { id: messageId, threadId },
+      labelIds,
+    } as gmail_v1.Schema$HistoryLabelRemoved,
+  });
+
+  const mockGmail = {
+    users: {
+      labels: {
+        list: vi.fn().mockImplementation((params) => {
+          console.log("Mock gmail.users.labels.list called with:", params);
+          return Promise.resolve({
+            data: {
+              labels: [
+                { id: "label-1", name: inboxZeroLabels.cold_email.name },
+                { id: "label-2", name: "Newsletter" },
+                { id: "label-3", name: "Marketing" },
+                { id: "label-4", name: "To Reply" },
+              ],
+            },
+          });
+        }),
+      },
+    },
+  } as any;
+  const mockEmailAccount = {
+    id: "email-account-id",
+    email: "user@test.com",
+  } as any;
+  const mockProvider = {
+    getMessage: vi.fn().mockResolvedValue({
+      headers: {
+        from: "sender@example.com",
+      },
+    }),
+  } as any;
+
+  const defaultOptions = {
+    gmail: mockGmail,
+    emailAccount: mockEmailAccount,
+    provider: mockProvider,
+  };
+
+  describe("handleLabelRemovedEvent", () => {
+    it("should process Cold Email label removal and update ColdEmail status", async () => {
+      prisma.coldEmail.upsert.mockResolvedValue({} as any);
+
+      const historyItem = createLabelRemovedHistoryItem();
+
+      console.log("Test data:", JSON.stringify(historyItem.item, null, 2));
+
+      try {
+        await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+      } catch (error) {
+        console.error("Function error:", error);
+        throw error;
+      }
+
+      expect(prisma.coldEmail.upsert).toHaveBeenCalledWith({
+        where: {
+          emailAccountId_fromEmail: {
+            emailAccountId: "email-account-id",
+            fromEmail: "sender@example.com",
+          },
+        },
+        update: {
+          status: ColdEmailStatus.USER_REJECTED_COLD,
+        },
+        create: {
+          status: ColdEmailStatus.USER_REJECTED_COLD,
+          fromEmail: "sender@example.com",
+          emailAccountId: "email-account-id",
+          messageId: "123",
+          threadId: "thread-123",
+        },
+      });
+    });
+
+    it("should skip learning when Newsletter label is removed (only Cold Email is supported)", async () => {
+      const historyItem = createLabelRemovedHistoryItem("123", "thread-123", [
+        "label-2",
+      ]);
+
+      await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+
+      expect(saveLearnedPatterns).not.toHaveBeenCalled();
+    });
+
+    it("should skip learning when To Reply label is removed (only Cold Email is supported)", async () => {
+      const historyItem = createLabelRemovedHistoryItem("123", "thread-123", [
+        "label-4",
+      ]);
+
+      await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+
+      expect(saveLearnedPatterns).not.toHaveBeenCalled();
+    });
+
+    it("should skip learning when no executed rule exists (only Cold Email is supported)", async () => {
+      const historyItem = createLabelRemovedHistoryItem("123", "thread-123", [
+        "label-2",
+      ]);
+
+      await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+
+      expect(saveLearnedPatterns).not.toHaveBeenCalled();
+    });
+
+    it("should skip learning when no matching LABEL action is found (only Cold Email is supported)", async () => {
+      const historyItem = createLabelRemovedHistoryItem("123", "thread-123", [
+        "label-2",
+      ]);
+
+      await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+
+      expect(saveLearnedPatterns).not.toHaveBeenCalled();
+    });
+
+    it("should handle multiple label removals in a single event (only Cold Email is supported)", async () => {
+      const historyItem = createLabelRemovedHistoryItem("123", "thread-123", [
+        "label-3",
+      ]);
+
+      await handleLabelRemovedEvent(historyItem.item, defaultOptions);
+
+      expect(saveLearnedPatterns).not.toHaveBeenCalled();
+    });
+
+    it("should skip processing when messageId is missing", async () => {
+      const historyItem = {
+        message: { threadId: "thread-123" }, // Missing messageId
+        labelIds: ["label-1"],
+      } as gmail_v1.Schema$HistoryLabelRemoved;
+
+      await handleLabelRemovedEvent(historyItem, defaultOptions);
+
+      expect(prisma.coldEmail.upsert).not.toHaveBeenCalled();
+    });
+
+    it("should skip processing when threadId is missing", async () => {
+      const historyItem = {
+        message: { id: "123" }, // Missing threadId
+        labelIds: ["label-1"],
+      } as gmail_v1.Schema$HistoryLabelRemoved;
+
+      await handleLabelRemovedEvent(historyItem, defaultOptions);
+
+      expect(prisma.coldEmail.upsert).not.toHaveBeenCalled();
+    });
+  });
+});
diff --git a/apps/web/app/api/google/webhook/process-label-removed-event.ts b/apps/web/app/api/google/webhook/process-label-removed-event.ts
new file mode 100644
index 0000000000..d02f5161c6
--- /dev/null
+++ b/apps/web/app/api/google/webhook/process-label-removed-event.ts
@@ -0,0 +1,142 @@
+import type { gmail_v1 } from "@googleapis/gmail";
+import prisma from "@/utils/prisma";
+
+import { ActionType, ColdEmailStatus } from "@prisma/client";
+import { logger } from "@/app/api/google/webhook/logger";
+import { extractEmailAddress } from "@/utils/email";
+import type { EmailAccountWithAI } from "@/utils/llms/types";
+import type { EmailProvider } from "@/utils/email/types";
+import { inboxZeroLabels } from "@/utils/label";
+import { GmailLabel } from "@/utils/gmail/label";
+import { saveLearnedPatterns } from "@/utils/rule/learned-patterns";
+
+const SYSTEM_LABELS = [
+  GmailLabel.INBOX,
+  GmailLabel.SENT,
+  GmailLabel.DRAFT,
+  GmailLabel.SPAM,
+  GmailLabel.TRASH,
+  GmailLabel.IMPORTANT,
+  GmailLabel.STARRED,
+  GmailLabel.UNREAD,
+];
+
+export async function handleLabelRemovedEvent(
+  message: gmail_v1.Schema$HistoryLabelRemoved,
+  {
+    gmail,
+    emailAccount,
+    provider,
+  }: {
+    gmail: gmail_v1.Gmail;
+    emailAccount: EmailAccountWithAI;
+    provider: EmailProvider;
+  },
+) {
+  const messageId = message.message?.id;
+  const threadId = message.message?.threadId;
+  const emailAccountId = emailAccount.id;
+  const userEmail = emailAccount.email;
+
+  const loggerOptions = {
+    email: userEmail,
+    messageId,
+    threadId,
+  };
+
+  if (!messageId || !threadId) {
+    logger.warn(
+      "Skipping label removal - missing messageId or threadId",
+      loggerOptions,
+    );
+    return;
+  }
+
+  logger.info("Processing label removal for learning", loggerOptions);
+
+  try {
+    const parsedMessage = await provider.getMessage(messageId);
+    const sender = extractEmailAddress(parsedMessage.headers.from);
+
+    const removedLabelIds = message.labelIds || [];
+
+    const labels = await gmail.users.labels.list({ userId: "me" });
+
+    const removedLabelNames = removedLabelIds
+      .map((labelId: string) => {
+        const label = labels.data.labels?.find(
+          (l: gmail_v1.Schema$Label) => l.id === labelId,
+        );
+        return label?.name;
+      })
+      .filter(
+        (labelName: string | null | undefined): labelName is string =>
+          !!labelName && !SYSTEM_LABELS.includes(labelName),
+      );
+
+    for (const labelName of removedLabelNames) {
+      await learnFromRemovedLabel({
+        labelName,
+        sender,
+        messageId,
+        threadId,
+        emailAccountId,
+      });
+    }
+  } catch (error) {
+    logger.error("Error processing label removal", { error, ...loggerOptions });
+  }
+}
+
+async function learnFromRemovedLabel({
+  labelName,
+  sender,
+  messageId,
+  threadId,
+  emailAccountId,
+}: {
+  labelName: string;
+  sender: string | null;
+  messageId: string;
+  threadId: string;
+  emailAccountId: string;
+}) {
+  const loggerOptions = {
+    emailAccountId,
+    messageId,
+    threadId,
+    labelName,
+    sender,
+  };
+
+  // Can't learn patterns without knowing who to exclude
+  if (!sender) {
+    logger.info("No sender found, skipping learning", loggerOptions);
+    return;
+  }
+
+  if (labelName === inboxZeroLabels.cold_email.name) {
+    logger.info("Processing Cold Email label removal", loggerOptions);
+
+    await prisma.coldEmail.upsert({
+      where: {
+        emailAccountId_fromEmail: {
+          emailAccountId,
+          fromEmail: sender,
+        },
+      },
+      update: {
+        status: ColdEmailStatus.USER_REJECTED_COLD,
+      },
+      create: {
+        status: ColdEmailStatus.USER_REJECTED_COLD,
+        fromEmail: sender,
+        emailAccountId,
+        messageId,
+        threadId,
+      },
+    });
+
+    return;
+  }
+}
diff --git a/apps/web/app/api/google/webhook/types.ts b/apps/web/app/api/google/webhook/types.ts
index 21d738cab7..b40365ca0b 100644
--- a/apps/web/app/api/google/webhook/types.ts
+++ b/apps/web/app/api/google/webhook/types.ts
@@ -3,6 +3,15 @@ import type { RuleWithActionsAndCategories } from "@/utils/types";
 import type { EmailAccountWithAI } from "@/utils/llms/types";
 import type { EmailAccount } from "@prisma/client";
 
+export const HistoryEventType = {
+  MESSAGE_ADDED: "messageAdded",
+  LABEL_ADDED: "labelAdded",
+  LABEL_REMOVED: "labelRemoved",
+} as const;
+
+export type HistoryEventType =
+  (typeof HistoryEventType)[keyof typeof HistoryEventType];
+
 export type ProcessHistoryOptions = {
   history: gmail_v1.Schema$History[];
   gmail: gmail_v1.Gmail;