How to find duplicate files from programmatically?

Question

I want to know how to find duplicate files from an Android phone.

Please help me to find duplicate files in my whole phone.

Thank you!

Answer 1

First Create Object Class

import java.io.Serializable;
import java.util.List;

public class Duplicate implements Serializable {

    private List<DuplicateFile> duplicateFiles;

    public Duplicate(List<DuplicateFile> duplicateFiles) {
        this.duplicateFiles = duplicateFiles;
    }

    public List<DuplicateFile> getDuplicateFiles() {
        return duplicateFiles;
    }

    public void setDuplicateFiles(List<DuplicateFile> duplicateFiles) {
        this.duplicateFiles = duplicateFiles;
    }

    @Override
    public String toString() {
        return "{" +
                "duplicateFiles=" + duplicateFiles +
                '}' + "\n";
    }
}

Another object class

import java.io.File;

public class DuplicateFile {

    public File file;
    public  boolean isSelect;

    public DuplicateFile(File file, boolean isSelect) {
        this.file = file;
        this.isSelect = isSelect;
    }

    public File getFile() {
        return file;
    }

    public void setFile(File file) {
        this.file = file;
    }

    public boolean isSelect() {
        return isSelect;
    }

    public void setSelect(boolean select) {
        isSelect = select;
    }

    @Override
    public String toString() {
        return "{" +
                "file=" + file +
                ", isSelect=" + isSelect +
                '}';
    }
}

Get All Files from storage for finding duplicate files

public static List<File> getAllFiles(Context context) {
        try {
            List<File> files = new ArrayList<>();

            String pdf = MimeTypeMap.getSingleton().getMimeTypeFromExtension("pdf");
            String doc = MimeTypeMap.getSingleton().getMimeTypeFromExtension("doc");
            String docx = MimeTypeMap.getSingleton().getMimeTypeFromExtension("docx");
            String xlsx = MimeTypeMap.getSingleton().getMimeTypeFromExtension("xlsx");
            String txt = MimeTypeMap.getSingleton().getMimeTypeFromExtension("txt");
            String ppt = MimeTypeMap.getSingleton().getMimeTypeFromExtension("ppt");
            String pptx = MimeTypeMap.getSingleton().getMimeTypeFromExtension("pptx");

            String png = MimeTypeMap.getSingleton().getMimeTypeFromExtension("png");
            String jpg = MimeTypeMap.getSingleton().getMimeTypeFromExtension("jpg");
            String jpeg = MimeTypeMap.getSingleton().getMimeTypeFromExtension("jpeg");

            String gif = MimeTypeMap.getSingleton().getMimeTypeFromExtension("gif");

            String mp4 = MimeTypeMap.getSingleton().getMimeTypeFromExtension("mp4");
            String mkv = MimeTypeMap.getSingleton().getMimeTypeFromExtension("mkv");
            String avi = MimeTypeMap.getSingleton().getMimeTypeFromExtension("avi");

            String apk = MimeTypeMap.getSingleton().getMimeTypeFromExtension("apk");
            String mp3 = MimeTypeMap.getSingleton().getMimeTypeFromExtension("mp3");

            Uri table = MediaStore.Files.getContentUri("external");

            String[] data = new String[]{MediaStore.Files.FileColumns.DATA};
            String where = MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?" + " OR "
                    + MediaStore.Files.FileColumns.MIME_TYPE + "=?";
            String[] args = new String[]{pdf, doc, docx, xlsx, txt, ppt, pptx, png, jpg, jpeg, gif, mp4, mkv, avi, apk, mp3};

            Cursor cursor = context.getContentResolver().query(table, data, where, args, null);

            if (cursor != null) {
                while (cursor.moveToNext()) {
                    File file = new File(cursor.getString(cursor.getColumnIndex(data[0])));
                    files.add(file);
                }
                cursor.close();
            }
            return files;
        } catch (Exception e) {
            return new ArrayList<>();
        }
    }

Now final method to find duplicate files

public static List<Duplicate> getDuplicateFiles(List<File> files) {
        HashMap<String, Duplicate> hashmap = new HashMap<>();
        HashMap<String, Duplicate> duplicateHashSet = new HashMap<>();
        for (File file : files) {
            String md5 = getFileMD5ToString(file);
            if (hashmap.containsKey(md5)) {
                Duplicate original = hashmap.get(md5);
                if (duplicateHashSet.containsKey(md5)) {
                    List<DuplicateFile> fileList;
                    if (original != null) {
                        fileList = original.getDuplicateFiles();
                        fileList.add(new DuplicateFile(original.getDuplicateFiles().get(0).getFile(), false));
                    } else {
                        fileList = new ArrayList<>();
                        fileList.add(new DuplicateFile(file, false));
                    }
                } else {
                    List<DuplicateFile> fileList = new ArrayList<>();
                    if (original.getDuplicateFiles() == null) {
                        original.setDuplicateFiles(new ArrayList<>());
                    }

                    fileList.add(new DuplicateFile(original.getDuplicateFiles().get(0).getFile(), false));
                    fileList.add(new DuplicateFile(file, false));
                    duplicateHashSet.put(md5, new Duplicate(fileList));
                }
            } else {
                List<DuplicateFile> fileList = new ArrayList<>();
                fileList.add(new DuplicateFile(file, false));
                hashmap.put(md5, new Duplicate(fileList));
            }
        }

        List<Duplicate> list = null;
        if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.N) {
            list = new ArrayList<>(duplicateHashSet.values());
        }
        Log.e(TAG, "getDuplicateFiles: ");
        return list;
    }

Note: You can find getFileMD5ToString() from below

implementation 'com.blankj:utilcodex:1.30.6'