C# Outlook interop add-in VSTO email delete deletion tracking

Question

I am putting together a C# VSTO Add-In for Outlook to yield the functionality of one of those "All Mailboxes" type folders that Microsoft utilized in earlier times. The idea is to grab everything from >= 2 inboxes and combine them in this All Mailboxes folder I've created. I'm having trouble tracking deleted emails by Entry ID, though - this relates to my idea of having "two-way" listening, so that emails deleted in the source inboxes or the All Mailboxes folder will delete the other as well.

There are several references to a "delete buffer" in my AddIn.cs file as it stands right now, but I'm moving in the direction of abandoning that notion of buffering in case of the user deleting emails en masse.

Part of the code that is commented out now:

allMailboxesItems.ItemRemove += new Outlook.ItemsEvents_ItemRemoveEventHandler(OnItemRemovedFromAllMailboxes);

...is in need of recalibrating to achieve the goals I mentioned earlier. The All Mailboxes folder that is the ultimate product of this Add-In needs to listen for deletions from its source folders so that it may update itself accordingly, and vice versa.

Anything that happens in one folder needs to happen in the other as well.

Here is the AddIn.cs file where all methods are collected together, until I move to the step of differentiating, grouping, and separating them. Please forgive the presence of some extraneous methods and variables.

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Xml.Linq;
using Outlook = Microsoft.Office.Interop.Outlook;
using Office = Microsoft.Office.Core;
using Microsoft.Office.Interop.Outlook;
using System.IO;
using Newtonsoft.Json;
using System.Windows.Forms;
using System.Diagnostics;
using System.ComponentModel;
using System.Threading.Tasks;

namespace AllMailboxes
{
    public partial class ThisAddIn
    {
        private System.Timers.Timer duplicateRemovalTimer;
        private string jsonFilePath = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "names.json");
        private List<string> emailGroupNames;
        private List<Outlook.Items> inboxItemsList = new List<Outlook.Items>();
        private HashSet<string> processedEmailIds = new HashSet<string>();
        //****^^^^ can delete processed var ? did the copiedEmail... HashSet totally take over its function?
        private HashSet<string> copiedEmailInventory = new HashSet<string>();
        private Dictionary<string, string> origEmails = new Dictionary<string, string>();
        //^^used with LinkCopyUniqueStringTo.. method to have connection b/w original emails and 
        private Dictionary<string, string> uniqueStringToEntryId = new Dictionary<string, string>();
        private List<string> buffer = new List<string>();
        private Queue<string> deleteBuffer = new Queue<string>();
        private Timer deleteBufferTimer;
        private Timer srcDeleteBufferTimer;
        private Outlook.Folder allMailboxesFolder;
        private void ThisAddIn_Startup(object sender, System.EventArgs e)
        {
            Outlook.Application outlookApp = this.Application;
            Outlook.NameSpace outlookNamespace = outlookApp.GetNamespace("MAPI");
            DateTime filterDateTime = DateTime.Now.AddDays(-7);

            LoadEmailGroupNamesFromJson();
            var primaryInbox = emailGroupNames.FirstOrDefault();

            Outlook.Folder primaryMailbox = outlookApp.Session.Folders[primaryInbox] as Outlook.Folder;
            allMailboxesFolder = GetUnifiedFolder(outlookApp);

            if (allMailboxesFolder == null)
            {
                allMailboxesFolder = primaryMailbox.Folders.Add("All Mailboxes", Outlook.OlDefaultFolders.olFolderInbox) as Outlook.Folder;
            }
            //^^create the All Mailboxes folder if it doesn't exist yet

            AllMailboxesInventory();

            foreach (var mailBoxName in emailGroupNames)
            {
                try
                {
                    Outlook.Folder mailbox = outlookApp.Session.Folders[mailBoxName] as Outlook.Folder;

                    if (mailbox != null)
                    {
                        Outlook.Folder inboxFolder = mailbox.Folders["Inbox"] as Outlook.Folder;
                        Outlook.Items inboxItems = inboxFolder.Items;

                        string filterDate = filterDateTime.ToString("MM/dd/yyyy");
                        string filter = $"[ReceivedTime] >= '{filterDate}'";

                        Outlook.Items mailItems = inboxFolder.Items;
                        Outlook.Items filteredMailItems = mailItems.Restrict(filter);

                        foreach (object item in filteredMailItems)
                            //this is probably where duplicate emails are getting added - modify this to 
                            //do the unique email linkage instead (dictionary with <string1> = **original
                            //email's (straight from source inbox) entry id, <string2> = unique string
                            //combination that can be taken from any folder, original or copy
                        {
                            if (item is Outlook.MailItem mailItem)
                            {
                                try
                                {
                                    Outlook.MailItem itemCopy = item as Outlook.MailItem;
                                    string uniqueStr = UniqueStr(itemCopy);
                                    if (!copiedEmailInventory.Contains(uniqueStr))
                                    //^^makes sure it's not pre-existing
                                    {
                                        copiedEmailInventory.Add(uniqueStr);
                                        mailItem.Copy().Move(allMailboxesFolder);
                                        BufferAndSaveMapping(uniqueStr, itemCopy.EntryID);
                                        TrackOriginalEmail(itemCopy);
                                        //^^only when it has checked the uniqueness tracking values
                                        //does it queue it up to be written to the txt tracking
                                        //file -- there is a system in place to prevent duplicates
                                        //in the persistent txt tracking file
                                    }
                                }
                                catch (System.Exception ex)
                                {
                                    System.Diagnostics.Debug.WriteLine($"Error moving email: {ex.Message}");
                                }
                            }
                        }
                    }
                }
                catch (System.Exception ex)
                {
                    MessageBox.Show($"Error accessing folders: {ex.Message}","Error", MessageBoxButtons.OK, MessageBoxIcon.Error);
                }
            }
            InitializeInboxListeners(outlookApp);
            ScanForDuplicateEmails(filterDateTime);
            CleanUpAllMailboxes(filterDateTime);
            StartDuplicateRemovalTimer(filterDateTime);
        }

        private void ThisAddIn_Shutdown(object sender, System.EventArgs e)
        {
            // Note: Outlook no longer raises this event. If you have code that 
            //    must run when Outlook shuts down, see https://go.microsoft.com/fwlink/?LinkId=506785
            FlushBufferToFile();
        }

        private void StartDuplicateRemovalTimer(DateTime filterDateTime)
        {
            duplicateRemovalTimer = new System.Timers.Timer(300000); // Every hour
            duplicateRemovalTimer.Elapsed += (sender, e) => ScanForDuplicateEmails(filterDateTime);
            duplicateRemovalTimer.Start();
        }

        //following method is for linking emails in All Mailboxes with their source inboxes
        private void TrackOriginalEmail(Outlook.MailItem origMailItem)
        {
            origEmails[UniqueStr(origMailItem)] = origMailItem.EntryID;
        }

        //initializeinboxlisteners is to sync when new emails are received in source inboxes
        private void InitializeInboxListeners(Outlook.Application outlookApp)
        {
            Outlook.Folder allMailboxesFolder = GetUnifiedFolder(outlookApp);
            foreach (var mailboxName in emailGroupNames)
            {
                try
                {
                    Outlook.Folder mailbox = Application.Session.Folders[mailboxName] as Outlook.Folder;
                    if (mailbox != null)
                    {
                        Outlook.Folder inboxFolder = mailbox.Folders["Inbox"] as Outlook.Folder;

                        //get items collection and add to list to keep reference
                        Outlook.Items inboxItems = inboxFolder.Items;


                        inboxItemsList.Add(inboxItems);

                        //attach event handler for new items - need async here !
                        inboxItems.ItemAdd += new Outlook.ItemsEvents_ItemAddEventHandler((object item) =>
                        {
                            if (item is Outlook.MailItem newMailItem)
                            {
                                try
                                {
                                    string entryId = TryGetEntryIdWithRetries(newMailItem, 3, 500);
                                    if (entryId != null && !processedEmailIds.Contains(entryId))
                                    {
                                        processedEmailIds.Add(entryId);
                                        copiedEmailInventory.Add(UniqueStr(newMailItem));
                                        origEmails[UniqueStr(newMailItem)] = newMailItem.EntryID;
                                        Outlook.MailItem copiedMailItem = newMailItem.Copy().Move(allMailboxesFolder);
                                    }
                                } catch (System.Exception ex)
                                {
                                    MessageBox.Show($"error copying: {ex.Message}", "****ERR****", MessageBoxButtons.OK, MessageBoxIcon.Error);
                                }
                            }
                        });
                    }
                } catch (System.Exception ex)
                {
                    System.Diagnostics.Debug.WriteLine($"Error setting up inbox listeners for {mailboxName}: {ex.Message}");
                }
            }
        }

        private string TryGetEntryIdWithRetries(Outlook.MailItem mailItem, int retryCount, int delayMilliseconds)
        {
            for (int i = 0; i < retryCount; i++)
            {
                try
                {
                    return mailItem.EntryID;
                }
                catch
                {
                    System.Threading.Thread.Sleep(delayMilliseconds);
                }
            }
            System.Diagnostics.Debug.WriteLine("EntryID could not be accessed after retries.");
            return null;
        }

        private void ConfirmOrigEmailAndCopiedEmailLinked(Outlook.MailItem origEmail, Outlook.MailItem copiedEmail)
        {
            if (!origEmails.ContainsValue(origEmail.EntryID))
            {
                TrackOriginalEmail(origEmail);
            }
        }

        private void ScanForDuplicateEmails(DateTime filterDateTime)
        {
            HashSet<string> uniqueEmails = new HashSet<string>();
            Dictionary<string, int> removedVals = new Dictionary<string, int>(); 

            foreach (Outlook.MailItem mailItem in allMailboxesFolder.Items)
            {
                try
                {
                    string emailKey = UniqueStr(mailItem);
                    //^^Entry IDs are NOT unique for copied emails! this is the best solution for a uniqueness check
                    //that DOES factor in being copied
                    if (uniqueEmails.Contains(emailKey))
                    {
                        mailItem.Delete();
                    }
                    else
                    {
                        uniqueEmails.Add(emailKey);
                    }
                } catch (System.Exception ex)
                {
                    System.Diagnostics.Debug.WriteLine($"Error processing email: {ex.Message}");
                }
            }
            if (removedVals.Count != 0)
            {
                string prefix = "...";
                foreach (KeyValuePair<string, int> kvp in removedVals)
                {
                    string lastTen = kvp.Key.Length > 10 ? kvp.Key.Substring(kvp.Key.Length - 10) : kvp.Key;
                    string res = prefix + lastTen;
                    Console.WriteLine(res);
                }
            }
        }

        private void CleanUpAllMailboxes(DateTime cutoff)
        {
            HashSet<string> uniqueEmails = new HashSet<string>();
            int deletedCount = 0;

            for (int i = allMailboxesFolder.Items.Count; i > 0; i--)
            {
                try
                {
                    Outlook.MailItem mailItem = allMailboxesFolder.Items[i] as Outlook.MailItem;
                    if (mailItem != null)
                    {
                        string emailKey = UniqueStr(mailItem);

                        if (uniqueEmails.Contains(emailKey))
                        {
                            mailItem.Delete();
                            deletedCount++;
                            continue;
                        }
                        else
                        {
                            uniqueEmails.Add(emailKey);
                        }
                        if (mailItem.ReceivedTime < cutoff)
                        {
                            mailItem.Delete();
                            deletedCount++;
                        }
                    }
                }
                catch (System.Exception ex)
                {
                    System.Diagnostics.Debug.WriteLine($"Error processing email: {ex.Message}");
                }
            }
            Console.WriteLine($"Cleanup complete. {deletedCount} emails deleted.");
        }

        //persistent method for tracking emails follows
        private void CopyEmailsAndTrackSource(Outlook.Folder sourceInbox)
        {
            foreach (object item in sourceInbox.Items)
            {
                if (item is Outlook.MailItem mailItem)
                {
                    try
                    {
                        string uniqueString = UniqueStr(mailItem);

                        Outlook.MailItem copiedMailItem = mailItem.Copy().Move(allMailboxesFolder);
                    } catch (System.Exception ex)
                    {
                        Console.WriteLine($"Error encountered: {ex.Message}");
                    }
                }
            }
        }

        private void BufferAndSaveMapping(string uniqueString, string entryId)
        {
            buffer.Add($"{uniqueString},{entryId}");

            if (buffer.Count >= 100)
            {
                FlushBufferToFile();
            }
        }

        private void FlushBufferToFile()
        {
            using (var writer = System.IO.File.AppendText("UniqueStringToEntryIDMap.txt"))
            {
                foreach (var line in buffer)
                {
                    writer.WriteLine(line);
                }
            }
            buffer.Clear(); // Clear the buffer after writing
        }

        private void SaveAllMappingsToFile(Dictionary<string, string> uniqueStringEntryId)
        {
            using (var writer = new System.IO.StreamWriter("UniqueStringToEntryIDMap.txt", append: true))
            {
                foreach (var entry in uniqueStringEntryId)
                {
                    writer.WriteLine($"{entry.Key},{entry.Value}");
                }
            }
        }

        private void LoadMappingFromFile()
        {
            if (System.IO.File.Exists("UniqueStringToEntryIDMap.txt"))
            {
                foreach (var line in System.IO.File.ReadAllLines("UniqueStringToEntryIDMap.txt"))
                {
                    var parts = line.Split(',');
                    if (parts.Length == 2)
                    {
                        uniqueStringToEntryId[parts[0]] = parts[1];
                    }
                }
            }
        }

        //private void SetUpAllMailboxesDeleteListener()
        //{
        //    Outlook.Items allMailboxesItems = allMailboxesFolder.Items;
        //    allMailboxesItems.ItemRemove += new Outlook.ItemsEvents_ItemRemoveEventHandler(OnItemRemovedFromAllMailboxes);

        //    deleteBufferTimer = new Timer(500);
        //    deleteBufferTimer.AutoReset = false;
        //    deleteBufferTimer.Elapsed += ProcessDeleteBufferQueue;
        //}

        //private void OnItemRemovedFromAllMailboxes()
        //{
        //    System.Threading.Tasks.Task.Delay(200).ContinueWith(t =>
        //    {
        //        string uniqueString = UniqueStr()
        //    })
        //}
        //^^11132024 WHERE I'M TRYING TO GET A UNIQUE MAILITEM - THE PRECEDING 2 METHODS
        //(SETUPALLMAILBOXES... AND ONITEMREMOVED...

        //private void SetUpSourceMailboxesDeleteListener(List<Outlook.Folder> otherFolders)
        //    //^^set up iterating through all source folders here, or do it when 
        //    //the method is invoked in a loop?
        //{
        //    foreach (Outlook.Folder otherFolder in otherFolders)
        //    {
        //        Outlook.Items otherFolderItems = otherFolder.Items;
        //        otherFolderItems.ItemRemove += new Outlook.ItemsEvents_ItemRemoveEventHandler(OnItemRemovedFromOtherFolder);

        //        srcDeleteBufferTimer = new Timer(500);
        //        srcDeleteBufferTimer.AutoReset = false;
        //        srcDeleteBufferTimer.Elapsed += ProcessDeleteBufferQueue;
        //    }
        //}

        //private void SetUpAllMailboxesDeleteListener()
        //{
        //    allMailboxesFolder.Items.ItemRemove += new Outlook.ItemsEvents_ItemRemoveEventHandler(OnItemRemovedFromAllMailboxes);
        //}
        //^^11132024 listener for all mailboxes folder AND its source folders (2 way listening)

        private void AllMailboxesInventory()
        {
            foreach(Outlook.MailItem mailItem in allMailboxesFolder.Items)
            {
                if (!copiedEmailInventory.Contains(UniqueStr(mailItem)))
                {
                    copiedEmailInventory.Add(UniqueStr(mailItem));
                }
            }
        }

        private string UniqueStr(Outlook.MailItem mailItem)
        {
            return $"{mailItem.Subject}-{mailItem.ReceivedTime}-{mailItem.SenderEmailAddress}".ToUpper(); ;
        }

        private void LoadEmailGroupNamesFromJson()
        {
            if ( (File.Exists(jsonFilePath))) {
                var jsonContent = File.ReadAllText(jsonFilePath);
                emailGroupNames = JsonConvert.DeserializeObject<List<string>>(jsonContent);
            }
            else {
                emailGroupNames = new List<string> { "Danyelle.McNeill@arkansas.gov", "ADH PhyNews", "ADH Communications" };
                SaveEmailGroupNamesToJson();
            }
        }

        private void SaveEmailGroupNamesToJson()
        {
            var jsonContent = JsonConvert.SerializeObject(emailGroupNames, Formatting.Indented);
            File.WriteAllText(jsonFilePath, jsonContent);
        }

        private void CreateContextMenuForAllMailboxes()
        {
            var allMailboxesFolder = Application.Session.Folders["All Mailboxes"] as Outlook.Folder;
            if (allMailboxesFolder == null) return;

            //allMailboxesFolder.BeforeFolderAdd += 
        }

        public void ShowPopupToEditEmailGroupNames()
        {
            using (var popupForm = new EmailGroupEditorForm(emailGroupNames))
            {
                if (popupForm.ShowDialog() == DialogResult.OK)
                {
                    emailGroupNames = popupForm.UpdatedEmailGroupNames;
                    SaveEmailGroupNamesToJson();
                    MessageBox.Show("Email group names updated successfully.");
                }
            }
        }

        private Outlook.Folder GetUnifiedFolder(Outlook.Application outlookApp)
        {
            var primaryInbox = emailGroupNames.FirstOrDefault();

            Outlook.Folder primaryMailbox = outlookApp.Session.Folders[primaryInbox] as Outlook.Folder;
            //^^sets All Mailboxes folder into user's personal email inbox (the 1st entry in json file)
            return primaryMailbox.Folders["All Mailboxes"] as Outlook.Folder;
        }
        #region VSTO generated code

        /// <summary>
        /// Required method for Designer support - do not modify
        /// the contents of this method with the code editor.
        /// </summary>
        private void InternalStartup()
        {
            this.Startup += new System.EventHandler(ThisAddIn_Startup);
            this.Shutdown += new System.EventHandler(ThisAddIn_Shutdown);
        }
        
        #endregion

    }
}