How do I search for visually duplicated images from my directory and move them?

Question

private void button4_Click(object sender, EventArgs e)
{
    FileStream outputFileStream = new FileStream("log.txt", FileMode.Create, FileAccess.Write);
    StreamWriter writer = new StreamWriter(outputFileStream);

    string originalPathFile = @"C:\Users\user\Downloads\CaptchaCollection\Small\";
    string duplicatePath = @"C:\Users\user\Downloads\CaptchaCollection\Small\Duplicates\";
    string movedOriginal = @"C:\Users\user\Downloads\CaptchaCollection\Small\Sorted\";

    bool endInner = false;
    bool endOuter = false;
    int count = 1;

    while (!endOuter)
    {
        if (Directory.GetFiles(originalPathFile).Length < 2)
        {
            endOuter = true;
        }

        while (!endInner)
        {
            var files = Directory.GetFiles(originalPathFile).Select(nameWithExtension => Path.GetFileNameWithoutExtension(nameWithExtension)).Where(name => { int number; return int.TryParse(name, out number); }).Select(name => int.Parse(name)).OrderBy(number => number).ToArray();

            Bitmap im1 = new Bitmap(originalPathFile + files[0].ToString() + ".png");
            Bitmap im2 = new Bitmap(originalPathFile + files[count].ToString() + ".png");

            count++;

            if (compare(im1, im2))
            {
                // if it's equal
                //File.Delete(originalPathFile + files[count2].ToString() + ".png");
                File.Move(originalPathFile + files[count].ToString() + ".png", duplicatePath + files[count].ToString() + ".png");
                writer.WriteLine(files[count] + ".png" + " is a duplicate of " + files[0] + ".png");
            }

            if (count >= files.Length - 1)
            {
                File.Move(originalPathFile + files[0].ToString() + ".png", movedOriginal + files[0].ToString() + ".png");
                writer.WriteLine(files[0] + ".png " + "has had its duplicates removed." );
                count = 1;

                endInner = true;
            }
        }
    }

    writer.Close();
    outputFileStream.Close();
}

This is my code for using the search loop to search directory and then check for visual duplicates.

private bool compare(Bitmap bmp1, Bitmap bmp2)
{
    bool equals = true;
    Rectangle rect = new Rectangle(0, 0, bmp1.Width, bmp1.Height);
    BitmapData bmpData1 = bmp1.LockBits(rect, ImageLockMode.ReadOnly, bmp1.PixelFormat);
    BitmapData bmpData2 = bmp2.LockBits(rect, ImageLockMode.ReadOnly, bmp2.PixelFormat);
    unsafe
    {
        byte* ptr1 = (byte*)bmpData1.Scan0.ToPointer();
        byte* ptr2 = (byte*)bmpData2.Scan0.ToPointer();
        int width = rect.Width * 3; // for 24bpp pixel data
        for (int y = 0; equals && y < rect.Height; y++)
        {
            for (int x = 0; x < width; x++)
            {
                if (*ptr1 != *ptr2)
                {
                    equals = false;
                    break;
                }
                ptr1++;
                ptr2++;
            }
            ptr1 += bmpData1.Stride - width;
            ptr2 += bmpData2.Stride - width;
        }
    }
    bmp1.UnlockBits(bmpData1);
    bmp2.UnlockBits(bmpData2);
    bmp1.Dispose();
    bmp2.Dispose();

    return equals;
}

This method is used to compare 2 images at a time if they are the same or not.

What I intended

The mainline logic here is supposed to select the first index of a directory file. Then it will compare that to every file in the directory to check the duplicates of that first images. After reaching the last file in the directory it should move the first index to another directory. Then it will continue until there are less than 2 files left.

What was not intended

It caused probably an infinite loop and it didn't write anything in my text file to check the move logs. Then after stopping the program, I checked the directories but only the first index was moved to another folder. Then I even checked the Duplicates folder but none of those files were duplicated. They were in fact all different.

In summary, is my loop really infinite and if so how could I fix that?

---

My Code after the answer

private void button4_Click(object sender, EventArgs e)
{
    FileStream outputFileStream = new FileStream("log.txt", FileMode.Create, FileAccess.Write);
    StreamWriter writer = new StreamWriter(outputFileStream);

    string originalPathFile = @"C:\Users\user\Downloads\CaptchaCollection\Small\";
    string duplicatePath = @"C:\Users\user\Downloads\CaptchaCollection\Small\Duplicates\";
    string movedOriginal = @"C:\Users\user\Downloads\CaptchaCollection\Small\Sorted\";

    foreach( String fileToCheck in Directory.GetFiles(originalPathFile,"*.png"))
    {
        foreach (String fileWithCheck in Directory.GetFiles(originalPathFile, "*.png"))
        {
            if (fileToCheck != fileWithCheck)
            {
                // Your code to compare...
                Bitmap im1 = new Bitmap(fileToCheck);
                Bitmap im2 = new Bitmap(fileWithCheck);   

                if (compare(im1, im2))
                {
                    // if it's equal
                    File.Move(originalPathFile + fileWithCheck + ".png", duplicatePath + fileWithCheck + ".png");
                    writer.WriteLine(fileWithCheck + ".png" + " is a duplicate of " + fileToCheck + ".png");
                }

            }
        }

        File.Move(originalPathFile + fileToCheck + ".png", movedOriginal + fileToCheck + ".png");
        writer.WriteLine(fileToCheck + ".png " + "has had its duplicates removed.");
    }

    writer.Close();
    outputFileStream.Close();
}

I'm getting a NotSupportedException on this line:

File.Move(originalPathFile + fileWithCheck + ".png", duplicatePath + fileWithCheck + ".png");

Answer 1

I'm not sure why your code enters an infinite loop, though I did notice that you're comparing the count of files checked against the number of files in the directory - and since the files you're checking are only those with a numeric name and a .png extension, if there are other files in that directory your comparison will be flawed.

The reason your updated code is throwing a NotSupportedException is because either the source filename or the destination filename are not in the correct format. Note that Directory.GetFiles() returns all the files in the specified directory, including their path - so you're duplicating the path when you use OriginalPathFile + fileToCheck + ".png" - in other words, if originalPath was "C:\Temp\ImageFiles\", for example, then what you're actually passing in as the file name is "C:\Temp\ImageFiles\C:\Temp\ImageFiles\" + fileToCheck + ".png".

Additionally, the collection in a foreach is immutable - you cannot alter it (add or remove items), so even if the File.Move method worked, you would have encountered an error on the next iteration.

The below code (which is not tested, but should at least give you something to start with) shows an alternative way that tries to stay true to your original intention.

var files = Directory.GetFiles(originalPathFile)
                     .Select(nameWithExtension => Path.GetFileNameWithoutExtension(nameWithExtension))
                     .Where(name => { int number; return int.TryParse(name, out number); })
                     .Select(name => int.Parse(name))
                     .OrderBy(number => number).ToArray();

while (files.Length > 1)
{

    for (int j = 1; j < files.Length; j++)
    {
        Bitmap im1 = new Bitmap(originalPathFile + files[0].ToString() + ".png");
        Bitmap im2 = new Bitmap(originalPathFile + files[j].ToString() + ".png");

        if (compare(im1, im2))
        {
            File.Move(originalPathFile + files[j].ToString() + ".png", duplicatePath + files[j].ToString() + ".png");
            writer.WriteLine(files[j].ToString() + ".png" + " is a duplicate of " + files[0].ToString() + ".png");
        }
    }

    File.Move(originalPathFile + files[0].ToString() + ".png", movedOriginal + files[0].ToString() + ".png");
    writer.WriteLine(files[0].ToString() + ".png " + "has had its duplicates removed." );

    files = Directory.GetFiles(originalPathFile)
                     .Select(nameWithExtension => Path.GetFileNameWithoutExtension(nameWithExtension))
                     .Where(name => { int number; return int.TryParse(name, out number); })
                     .Select(name => int.Parse(name))
                     .OrderBy(number => number).ToArray();
}

In the above code, the list of files is retrieved initially and put into var files.

The outer while loop will execute as long as there is more than one file in the directory. In the while loop a for loop is used to go through the list of files (starting at the 2nd element - notice j = 1) and does the comparisons. If a match is found, that that file is moved to the duplicates folder. Unlike a foreach loop, the contents of the collection can be modified in a for loop.

After the for loop is completed, the original file is moved, and the loop continues until there is only one file left in the list of files that match what you are looking for. There is another call to get the list of files as well, since it may have been modified and you would get exceptions for the files that were no longer there.

You could then move that file after the while loop is completed, if desired.

There are undoubtedly improvements that could be made (I'm thinking LINQ would work here), but this will give you something to try at least.