//
// ========================================================================
// Copyright (c) 1995-2020 Mort Bay Consulting Pty Ltd and others.
//
// This program and the accompanying materials are made available under
// the terms of the Eclipse Public License 2.0 which is available at
// https://www.eclipse.org/legal/epl-2.0
//
// This Source Code may also be made available under the following
// Secondary Licenses when the conditions for such availability set
// forth in the Eclipse Public License, v. 2.0 are satisfied:
// the Apache License v2.0 which is available at
// https://www.apache.org/licenses/LICENSE-2.0
//
// SPDX-License-Identifier: EPL-2.0 OR Apache-2.0
// ========================================================================
//

package org.eclipse.jetty.util;

import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.FileVisitOption;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.PathMatcher;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Predicate;

import org.eclipse.jetty.util.component.AbstractLifeCycle;
import org.eclipse.jetty.util.thread.ScheduledExecutorScheduler;
import org.eclipse.jetty.util.thread.Scheduler;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

Scanner Utility for scanning a directory for added, removed and changed files and reporting these events via registered Listeners.
/** * Scanner * * Utility for scanning a directory for added, removed and changed * files and reporting these events via registered Listeners. */
public class Scanner extends AbstractLifeCycle {
When walking a directory, a depth of 1 ensures that the directory's descendants are visited, not just the directory itself (as a file).
/** * When walking a directory, a depth of 1 ensures that * the directory's descendants are visited, not just the * directory itself (as a file). */
public static final int DEFAULT_SCAN_DEPTH = 1; public static final int MAX_SCAN_DEPTH = Integer.MAX_VALUE; private static final Logger LOG = LoggerFactory.getLogger(Scanner.class); private static final AtomicInteger SCANNER_IDS = new AtomicInteger(); private int _scanInterval; private final AtomicInteger _scanCount = new AtomicInteger(0); private final List<Listener> _listeners = new CopyOnWriteArrayList<>(); private Map<String, MetaData> _prevScan; private FilenameFilter _filter; private final Map<Path, IncludeExcludeSet<PathMatcher, Path>> _scannables = new ConcurrentHashMap<>(); private boolean _reportExisting = true; private boolean _reportDirs = true; private Scheduler.Task _task; private Scheduler _scheduler; private int _scanDepth = DEFAULT_SCAN_DEPTH; private enum Status { ADDED, CHANGED, REMOVED, STABLE } enum Notification { ADDED, CHANGED, REMOVED }
PathMatcherSet A set of PathMatchers for testing Paths against path matching patterns via
See Also:
  • IncludeExcludeSet
/** * PathMatcherSet * * A set of PathMatchers for testing Paths against path matching patterns via * @see IncludeExcludeSet */
static class PathMatcherSet extends HashSet<PathMatcher> implements Predicate<Path> { @Override public boolean test(Path p) { for (PathMatcher pm : this) { if (pm.matches(p)) return true; } return false; } }
MetaData Metadata about a file: Last modified time, file size and last file status (ADDED, CHANGED, DELETED, STABLE)
/** * MetaData * * Metadata about a file: Last modified time, file size and * last file status (ADDED, CHANGED, DELETED, STABLE) */
private static class MetaData { final long _lastModified; final long _size; Status _status; public MetaData(long lastModified, long size) { _lastModified = lastModified; _size = size; } public boolean isModified(MetaData m) { return m._lastModified != _lastModified || m._size != _size; } @Override public String toString() { return "[lm=" + _lastModified + ",sz=" + _size + ",s=" + _status + "]"; } } private class ScanTask implements Runnable { @Override public void run() { scan(); schedule(); } }
Visitor A FileVisitor for walking a subtree of paths. The Scanner uses this to examine the dirs and files it has been asked to scan.
/** * Visitor * * A FileVisitor for walking a subtree of paths. The Scanner uses * this to examine the dirs and files it has been asked to scan. */
private class Visitor implements FileVisitor<Path> { Map<String, MetaData> scanInfoMap; IncludeExcludeSet<PathMatcher,Path> rootIncludesExcludes; Path root; public Visitor(Path root, IncludeExcludeSet<PathMatcher,Path> rootIncludesExcludes, Map<String, MetaData> scanInfoMap) { this.root = root; this.rootIncludesExcludes = rootIncludesExcludes; this.scanInfoMap = scanInfoMap; } @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { if (!Files.exists(dir)) return FileVisitResult.SKIP_SUBTREE; File f = dir.toFile(); //if we want to report directories and we haven't already seen it if (_reportDirs && !scanInfoMap.containsKey(f.getCanonicalPath())) { boolean accepted = false; if (rootIncludesExcludes != null && !rootIncludesExcludes.isEmpty()) { //accepted if not explicitly excluded and either is explicitly included or there are no explicit inclusions accepted = rootIncludesExcludes.test(dir); } else { if (_filter == null || _filter.accept(f.getParentFile(), f.getName())) accepted = true; } if (accepted) { scanInfoMap.put(f.getCanonicalPath(), new MetaData(f.lastModified(), f.isDirectory() ? 0 : f.length())); if (LOG.isDebugEnabled()) LOG.debug("scan accepted dir {} mod={}", f, f.lastModified()); } } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { if (!Files.exists(file)) return FileVisitResult.CONTINUE; File f = file.toFile(); boolean accepted = false; if (f.isFile() || (f.isDirectory() && _reportDirs && !scanInfoMap.containsKey(f.getCanonicalPath()))) { if (rootIncludesExcludes != null && !rootIncludesExcludes.isEmpty()) { //accepted if not explicitly excluded and either is explicitly included or there are no explicit inclusions accepted = rootIncludesExcludes.test(file); } else if (_filter == null || _filter.accept(f.getParentFile(), f.getName())) accepted = true; } if (accepted) { scanInfoMap.put(f.getCanonicalPath(), new MetaData(f.lastModified(), f.isDirectory() ? 0 : f.length())); if (LOG.isDebugEnabled()) LOG.debug("scan accepted {} mod={}", f, f.lastModified()); } return FileVisitResult.CONTINUE; } @Override public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { LOG.warn("FileVisit failed: {}", file, exc); return FileVisitResult.CONTINUE; } @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { return FileVisitResult.CONTINUE; } }
Listener Marker for notifications re file changes.
/** * Listener * * Marker for notifications re file changes. */
public interface Listener { }
Notification of exact file changes in the last scan.
/** * Notification of exact file changes in the last scan. */
public interface DiscreteListener extends Listener { public void fileChanged(String filename) throws Exception; public void fileAdded(String filename) throws Exception; public void fileRemoved(String filename) throws Exception; }
Notification of files that changed in the last scan.
/** * Notification of files that changed in the last scan. */
public interface BulkListener extends Listener { public void filesChanged(Set<String> filenames) throws Exception; }
Listener that notifies when a scan has started and when it has ended.
/** * Listener that notifies when a scan has started and when it has ended. */
public interface ScanCycleListener extends Listener { public default void scanStarted(int cycle) throws Exception { } public default void scanEnded(int cycle) throws Exception { } } public Scanner() { }
Get the scan interval
Returns:interval between scans in seconds
/** * Get the scan interval * * @return interval between scans in seconds */
public int getScanInterval() { return _scanInterval; }
Set the scan interval
Params:
  • scanInterval – pause between scans in seconds, or 0 for no scan after the initial scan.
/** * Set the scan interval * * @param scanInterval pause between scans in seconds, or 0 for no scan after the initial scan. */
public void setScanInterval(int scanInterval) { if (isRunning()) throw new IllegalStateException("Scanner started"); _scanInterval = scanInterval; } public void setScanDirs(List<File> dirs) { if (isRunning()) throw new IllegalStateException("Scanner started"); _scannables.clear(); if (dirs == null) return; for (File f :dirs) { if (f.isDirectory()) addDirectory(f.toPath()); else addFile(f.toPath()); } }
Add a file to be scanned. The file must not be null, and must exist.
Params:
  • p – the Path of the file to scan.
/** * Add a file to be scanned. The file must not be null, and must exist. * * @param p the Path of the file to scan. */
public void addFile(Path p) { if (isRunning()) throw new IllegalStateException("Scanner started"); if (p == null) throw new IllegalStateException("Null path"); if (!Files.exists(p) || Files.isDirectory(p)) throw new IllegalStateException("Not file or doesn't exist: " + p); try { _scannables.putIfAbsent(p.toRealPath(), new IncludeExcludeSet<>(PathMatcherSet.class)); } catch (IOException e) { throw new IllegalStateException(e); } }
Add a directory to be scanned. The directory must not be null and must exist.
Params:
  • p – the directory to scan.
Returns:an IncludeExcludeSet to which the caller can add PathMatcher patterns to match
/** * Add a directory to be scanned. The directory must not be null and must exist. * * @param p the directory to scan. * @return an IncludeExcludeSet to which the caller can add PathMatcher patterns to match */
public IncludeExcludeSet<PathMatcher, Path> addDirectory(Path p) { if (isRunning()) throw new IllegalStateException("Scanner started"); if (p == null) throw new IllegalStateException("Null path"); if (!Files.exists(p) || !Files.isDirectory(p)) throw new IllegalStateException("Not directory or doesn't exist: " + p); try { IncludeExcludeSet<PathMatcher, Path> includesExcludes = new IncludeExcludeSet<>(PathMatcherSet.class); IncludeExcludeSet<PathMatcher, Path> prev = _scannables.putIfAbsent(p.toRealPath(), includesExcludes); if (prev != null) includesExcludes = prev; return includesExcludes; } catch (IOException e) { throw new IllegalStateException(e); } }
Apply a filter to files found in the scan directory. Only files matching the filter will be reported as added/changed/removed.
Params:
  • filter – the filename filter to use
/** * Apply a filter to files found in the scan directory. * Only files matching the filter will be reported as added/changed/removed. * * @param filter the filename filter to use */
@Deprecated public void setFilenameFilter(FilenameFilter filter) { _filter = filter; }
Get any filter applied to files in the scan dir.
Returns:the filename filter
/** * Get any filter applied to files in the scan dir. * * @return the filename filter */
@Deprecated public FilenameFilter getFilenameFilter() { return _filter; } public Set<Path> getScannables() { return Collections.unmodifiableSet(_scannables.keySet()); }
Get the scanDepth.
Returns:the scanDepth
/** * Get the scanDepth. * * @return the scanDepth */
public int getScanDepth() { return _scanDepth; }
Set the scanDepth.
Params:
  • scanDepth – the scanDepth to set
/** * Set the scanDepth. * * @param scanDepth the scanDepth to set */
public void setScanDepth(int scanDepth) { if (isRunning()) throw new IllegalStateException("Scanner started"); _scanDepth = scanDepth; }
Whether or not an initial scan will report all files as being added.
Params:
  • reportExisting – if true, all files found on initial scan will be reported as being added, otherwise not
/** * Whether or not an initial scan will report all files as being * added. * * @param reportExisting if true, all files found on initial scan will be * reported as being added, otherwise not */
public void setReportExistingFilesOnStartup(boolean reportExisting) { if (isRunning()) throw new IllegalStateException("Scanner started"); _reportExisting = reportExisting; } public boolean getReportExistingFilesOnStartup() { return _reportExisting; }
Set if found directories should be reported.
Params:
  • dirs – true to report directory changes as well
/** * Set if found directories should be reported. * * @param dirs true to report directory changes as well */
public void setReportDirs(boolean dirs) { if (isRunning()) throw new IllegalStateException("Scanner started"); _reportDirs = dirs; } public boolean getReportDirs() { return _reportDirs; }
Add an added/removed/changed listener
Params:
  • listener – the listener to add
/** * Add an added/removed/changed listener * * @param listener the listener to add */
public void addListener(Listener listener) { if (listener == null) return; _listeners.add(listener); }
Remove a registered listener
Params:
  • listener – the Listener to be removed
/** * Remove a registered listener * * @param listener the Listener to be removed */
public void removeListener(Listener listener) { if (listener == null) return; _listeners.remove(listener); }
Start the scanning action.
/** * Start the scanning action. */
@Override public void doStart() throws Exception { if (LOG.isDebugEnabled()) LOG.debug("Scanner start: rprtExists={}, depth={}, rprtDirs={}, interval={}, filter={}, scannables={}", _reportExisting, _scanDepth, _reportDirs, _scanInterval, _filter, _scannables); if (_reportExisting) { // if files exist at startup, report them scan(); scan(); // scan twice so files reported as stable } else { //just register the list of existing files and only report changes _prevScan = scanFiles(); } //Create the scheduler and start it _scheduler = new ScheduledExecutorScheduler("Scanner-" + SCANNER_IDS.getAndIncrement(), true, 1); _scheduler.start(); //schedule the scan schedule(); } private void schedule() { if (isRunning() && getScanInterval() > 0) _task = _scheduler.schedule(new ScanTask(), 1010L * getScanInterval(), TimeUnit.MILLISECONDS); }
Stop the scanning.
/** * Stop the scanning. */
@Override public void doStop() throws Exception { Scheduler.Task task = _task; _task = null; if (task != null) task.cancel(); Scheduler scheduler = _scheduler; _scheduler = null; if (scheduler != null) scheduler.stop(); }
Clear the list of scannables. The scanner must first be in the stopped state.
/** * Clear the list of scannables. The scanner must first * be in the stopped state. */
public void reset() { if (!isStopped()) throw new IllegalStateException("Not stopped"); //clear the scannables _scannables.clear(); //clear the previous scans _prevScan = null; }
Params:
  • path – tests if the path exists
Returns:true if the path exists in one of the scandirs
/** * @param path tests if the path exists * @return true if the path exists in one of the scandirs */
public boolean exists(String path) { for (Path p : _scannables.keySet()) { if (p.resolve(path).toFile().exists()) return true; } return false; }
Hint to the scanner to perform a scan cycle as soon as possible. NOTE that the scan is not guaranteed to have happened by the time this method returns.
/** * Hint to the scanner to perform a scan cycle as soon as possible. * NOTE that the scan is not guaranteed to have happened by the * time this method returns. */
public void nudge() { if (!isRunning()) throw new IllegalStateException("Scanner not running"); scan(Callback.NOOP); }
Get the scanner to perform a scan cycle as soon as possible and call the Callback when the scan is finished or failed.
Params:
  • complete – called when the scan cycle finishes or fails.
/** * Get the scanner to perform a scan cycle as soon as possible * and call the Callback when the scan is finished or failed. * * @param complete called when the scan cycle finishes or fails. */
public void scan(Callback complete) { Scheduler scheduler = _scheduler; if (!isRunning() || scheduler == null) { complete.failed(new IllegalStateException("Scanner not running")); return; } scheduler.schedule(() -> { try { scan(); complete.succeeded(); } catch (Throwable t) { complete.failed(t); } }, 0, TimeUnit.MILLISECONDS); }
Perform a pass of the scanner and report changes
/** * Perform a pass of the scanner and report changes */
void scan() { int cycle = _scanCount.incrementAndGet(); reportScanStart(cycle); Map<String, MetaData> currentScan = scanFiles(); reportDifferences(currentScan, _prevScan == null ? Collections.emptyMap() : Collections.unmodifiableMap(_prevScan)); _prevScan = currentScan; reportScanEnd(cycle); }
Scan all of the given paths.
/** * Scan all of the given paths. */
private Map<String, MetaData> scanFiles() { Map<String, MetaData> currentScan = new HashMap<>(); for (Path p : _scannables.keySet()) { try { Files.walkFileTree(p, EnumSet.allOf(FileVisitOption.class),_scanDepth, new Visitor(p, _scannables.get(p), currentScan)); } catch (IOException e) { LOG.warn("Error scanning files.", e); } } return currentScan; }
Report the adds/changes/removes to the registered listeners Only report an add or change once a file has stablilized in size.
Params:
  • currentScan – the info from the most recent pass
  • oldScan – info from the previous pass
/** * Report the adds/changes/removes to the registered listeners * * Only report an add or change once a file has stablilized in size. * * @param currentScan the info from the most recent pass * @param oldScan info from the previous pass */
private void reportDifferences(Map<String, MetaData> currentScan, Map<String, MetaData> oldScan) { Map<String, Notification> changes = new HashMap<>(); //Handle deleted files Set<String> oldScanKeys = new HashSet<>(oldScan.keySet()); oldScanKeys.removeAll(currentScan.keySet()); for (String file : oldScanKeys) { changes.put(file, Notification.REMOVED); } // Handle new and changed files for (String file : currentScan.keySet()) { MetaData current = currentScan.get(file); MetaData previous = oldScan.get(file); if (previous == null) { //New file - don't immediately //notify this, wait until the size has //settled down then notify the add. current._status = Status.ADDED; } else if (current.isModified(previous)) { //Changed file - handle case where file //that was added on previous scan has since //been modified. We need to retain status //as added, so we send the ADDED event once //the file has settled down. if (previous._status == Status.ADDED) current._status = Status.ADDED; else current._status = Status.CHANGED; } else { //Unchanged file: if it was previously //ADDED, we can now send the ADDED event. if (previous._status == Status.ADDED) changes.put(file, Notification.ADDED); else if (previous._status == Status.CHANGED) changes.put(file, Notification.CHANGED); current._status = Status.STABLE; } } if (LOG.isDebugEnabled()) LOG.debug("scanned {}", _scannables.keySet()); //Call the DiscreteListeners for (Map.Entry<String, Notification> entry : changes.entrySet()) { switch (entry.getValue()) { case ADDED: reportAddition(entry.getKey()); break; case CHANGED: reportChange(entry.getKey()); break; case REMOVED: reportRemoval(entry.getKey()); break; default: LOG.warn("Unknown file change: {}", entry.getValue()); break; } } //Call the BulkListeners reportBulkChanges(changes.keySet()); } private void warn(Object listener, String filename, Throwable th) { LOG.warn("{} failed on '{}'", listener, filename, th); }
Report a file addition to the registered FileAddedListeners
Params:
  • filename – the filename
/** * Report a file addition to the registered FileAddedListeners * * @param filename the filename */
private void reportAddition(String filename) { for (Listener l : _listeners) { try { if (l instanceof DiscreteListener) ((DiscreteListener)l).fileAdded(filename); } catch (Throwable e) { warn(l, filename, e); } } }
Report a file removal to the FileRemovedListeners
Params:
  • filename – the filename
/** * Report a file removal to the FileRemovedListeners * * @param filename the filename */
private void reportRemoval(String filename) { for (Object l : _listeners) { try { if (l instanceof DiscreteListener) ((DiscreteListener)l).fileRemoved(filename); } catch (Throwable e) { warn(l, filename, e); } } }
Report a file change to the FileChangedListeners
Params:
  • filename – the filename
/** * Report a file change to the FileChangedListeners * * @param filename the filename */
private void reportChange(String filename) { if (filename == null) return; for (Listener l : _listeners) { try { if (l instanceof DiscreteListener) ((DiscreteListener)l).fileChanged(filename); } catch (Throwable e) { warn(l, filename, e); } } }
Report the list of filenames for which changes were detected.
Params:
  • filenames – names of all files added/changed/removed
/** * Report the list of filenames for which changes were detected. * * @param filenames names of all files added/changed/removed */
private void reportBulkChanges(Set<String> filenames) { if (filenames == null || filenames.isEmpty()) return; for (Listener l : _listeners) { try { if (l instanceof BulkListener) ((BulkListener)l).filesChanged(filenames); } catch (Throwable e) { warn(l, filenames.toString(), e); } } }
Call ScanCycleListeners with start of scan
Params:
  • cycle – scan count
/** * Call ScanCycleListeners with start of scan * * @param cycle scan count */
private void reportScanStart(int cycle) { for (Listener listener : _listeners) { try { if (listener instanceof ScanCycleListener) ((ScanCycleListener)listener).scanStarted(cycle); } catch (Exception e) { LOG.warn("{} failed on scan start for cycle {}", listener, cycle, e); } } }
Call ScanCycleListeners with end of scan.
Params:
  • cycle – scan count
/** * Call ScanCycleListeners with end of scan. * * @param cycle scan count */
private void reportScanEnd(int cycle) { for (Listener listener : _listeners) { try { if (listener instanceof ScanCycleListener) ((ScanCycleListener)listener).scanEnded(cycle); } catch (Exception e) { LOG.warn("{} failed on scan end for cycle {}", listener, cycle, e); } } } }