Speeding up reading of multiple files and comparing their contents to multiple strings

Question

Problem: I have an array of some 700 strings, which I'm reading into a List. I then have a directory containing over 1500 files. I need to open every one of these files and see if any of the 700 strings appear anywhere within each of them.

Current solution: After reading in the 700 strings (which is pretty much instantaneous), this is what I'm doing:

public static void scanMyDirectory(final File myDirectory, final List<String> listOfStrings) {
    for (final File fileEntry : myDirectory.listFiles()) {
        System.out.println("Entering file: " + currentCount++);
        if (fileEntry.isDirectory()) {
            scanMyDirectory(fileEntry, listOfStrings);
        } else {

            BufferedReader br = null;
            try {
                String sCurrentLine;
                br = new BufferedReader(new FileReader(fileEntry.getPath()));

                while ((sCurrentLine = br.readLine()) != null) {
                    for (int i = 0; i < listOfStrings.size(); i++) {
                        if (org.apache.commons.lang3.StringUtils.containsIgnoreCase(sCurrentLine, listOfStrings.get(i))) {
                            matchLocations.put(listOfStrings.get(i), fileEntry.getPath());
                        }
                    }
                }
            } catch (IOException e) {
                e.printStackTrace();
            } finally {
                try {
                    if (br != null) {
                        br.close();
                    }
                } catch (IOException ex) {
                    ex.printStackTrace();
                }
            }
        }
    }
}

After calling this procedure, I have all the results stored in a HashMap and I can output the results to screen or file.

Question: What is the faster way to do this? It seems extremely slow (taking around 20-25 minutes to run through ~1500 files). I'm not very familiar with threading, but I've considered using it. However, the top answer in this question has put me off a little. What is the best way to speed up performance?

Answer 1

I prefer NIO to read lines :

private final Map<String, String> matchLocations = new HashMap<>();
private int currentCount = 0;

public void scanMyDirectory(final File myDirectory, final List<String> listOfStrings) {
    File[] files = myDirectory.listFiles();
    if (files == null) {
        return;
    }
    Stream.of(files).forEach(fileEntry -> {
        if (fileEntry.isDirectory()) {
            scanMyDirectory(fileEntry, listOfStrings);
        } else {
            System.out.println("Entering file: " + currentCount++);
            try {
                List<String> lines = Files.readAllLines(Paths.get(fileEntry.getAbsolutePath()), StandardCharsets.UTF_8);
                StringBuilder sb = new StringBuilder();
                lines.forEach(s -> sb.append(s.toLowerCase()).append("\n"));
                listOfStrings.forEach(s -> {
                    if (sb.indexOf(s.toLowerCase()) > 0) {
                        matchLocations.put(s, fileEntry.getPath());
                    }
                });
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
    });
}

as mentioned above, there's no need to multithreading...but if you're interested :

private final ConcurrentHashMap<String, String> matchLocations = new ConcurrentHashMap<>();
private final ForkJoinPool pool = new ForkJoinPool();
private int currentCount = 0;

public void scanMyDirectory(final File myDirectory, final List<String> listOfStrings) {
    File[] files = myDirectory.listFiles();
    if (files == null) {
        return;
    }
    Stream.of(files).forEach(fileEntry -> {
        if (fileEntry.isDirectory()) {
            scanMyDirectory(fileEntry, listOfStrings);
        } else {
            System.out.println("Entering file: " + currentCount++);
            pool.submit(new Reader(listOfStrings, fileEntry));
        }
    });
}

class Reader implements Runnable {

    final List<String> listOfStrings;
    final File file;

    Reader(List<String> listOfStrings, File file) {
        this.listOfStrings = listOfStrings;
        this.file = file;
    }

    @Override
    public void run() {
        try {
            List<String> lines = Files.readAllLines(Paths.get(file.getAbsolutePath()), StandardCharsets.UTF_8);
            StringBuilder sb = new StringBuilder();
            lines.forEach(s -> sb.append(s.toLowerCase()).append("\n"));
            listOfStrings.forEach(s -> {
                if (sb.indexOf(s.toLowerCase()) > 0) {
                    matchLocations.put(s, file.getPath());
                }
            });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}

EDIT

bug fixes :

private final ConcurrentHashMap<String, List<String>> matchLocations = new ConcurrentHashMap<>();
private final ForkJoinPool pool = new ForkJoinPool();
private int currentCount = 0;

public void scanMyDirectory(final File myDirectory, final List<String> listOfStrings) {
    File[] files = myDirectory.listFiles();
    if (files == null) {
        return;
    }
    Stream.of(files).forEach(fileEntry -> {
        if (fileEntry.isDirectory()) {
            scanMyDirectory(fileEntry, listOfStrings);
        } else {
            System.out.println("Entering file: " + currentCount++);
            Reader reader = new Reader(listOfStrings, fileEntry);
            pool.submit(reader);
        }
    });
}

class Reader implements Runnable {

    final List<String> listOfStrings;
    final File file;

    Reader(List<String> listOfStrings, File file) {
        this.listOfStrings = listOfStrings;
        this.file = file;
    }

    @Override
    public void run() {
        try (FileInputStream fileInputStream = new FileInputStream(file);
             FileChannel channel = fileInputStream.getChannel()) {
            StringBuilder sb = new StringBuilder();
            ByteBuffer buffer = ByteBuffer.allocate(512);
            int read;
            while (true) {
                read = channel.read(buffer);
                if (read == -1) {
                    break;
                }
                buffer.flip();
                sb.append(new String(buffer.array()).toLowerCase());
                buffer.clear();
            }
            listOfStrings.stream()
                    .map(String::toLowerCase)
                    .forEach(s -> {
                        if (sb.indexOf(s) > 0) {
                            List<String> current = matchLocations.get(s);
                            if (current == null) {
                                current = new ArrayList<>();
                                matchLocations.put(s, current);
                            }
                            current.add(file.getAbsolutePath());
                        }
                    });
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

}