flume taildir二次开发 支持多级目录检索

tairidr 支持多级目录检索 flume v1.9.0

• 位置: package org.apache.flume.source.taildir;
• 修改的方法: getMatchingFilesNoCache

//获取匹配的flie
List<File> getMatchingFiles() {
    long now = TimeUnit.SECONDS.toMillis(
        TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()));
    long currentParentDirMTime = parentDir.lastModified();
    List<File> result;

    // calculate matched files if
    // - we don't want to use cache (recalculate every time) OR
    // - directory was clearly updated after the last check OR
    // - last mtime change wasn't already checked for sure
    //   (system clock hasn't passed that second yet)
    if (!cachePatternMatching ||
        lastSeenParentDirMTime < currentParentDirMTime ||
        !(currentParentDirMTime < lastCheckedTime)) {
        //替换成自定义方法 getMatchingFilesNoCache
      lastMatchedFiles = sortByLastModifiedTime(getMatchingFilesNoCache());
      lastSeenParentDirMTime = currentParentDirMTime;
      lastCheckedTime = now;
    }

    return lastMatchedFiles;
  }
  
#源码 不支持 多级目录检索 文件 --改写 使用files.walkFileTree
 private List<File> getMatchingFilesNoCache() {
    List<File> result = Lists.newArrayList();
    try (DirectoryStream<Path> stream = Files.newDirectoryStream(parentDir.toPath(), fileFilter)) {
      for (Path entry : stream) {
        result.add(entry.toFile());
      }
    } catch (IOException e) {
      logger.error("I/O exception occurred while listing parent directory. " +
                   "Files already matched will be returned. " + parentDir.toPath(), e);
    }
    return result;
  }
  

• 自定义 实现taildir多级检索

 /**
   * 自定义 实现 taildir多级目录检索 使用walkFileTree
   * @return
   */
  private List<File> getMatchingFilesNoCache(Boolean isrecursion) {
    //配置中传入参数判断是否启用多级检索
    if (!isrecursion){
      return getMatchingFilesNoCache();
    }
    List<File> result = Lists.newArrayList();
    try{
      Files.walkFileTree(parentDir.toPath(),new SimpleFileVisitor<Path>(){
        @Override
        public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs){
          //获取目录
          try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, fileFilter)) {
            for (Path entry : stream) {
              result.add(entry.toFile());
            }
          } catch (IOException e) {
            logger.error("I/O exception occurred while listing parent directory. " +
                    "Files already matched will be returned. " + dir, e);
          }

          return FileVisitResult.CONTINUE;
        }

      });
    }catch (IOException e){
      logger.error("I/O exception occurred while listing parent directory. " +
              "Files already matched will be returned. " + parentDir.toPath(), e);
    }

    return result;
  }

• 修整的代码

List<File> getMatchingFiles() {
    long now = TimeUnit.SECONDS.toMillis(
        TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()));
    long currentParentDirMTime = parentDir.lastModified();
    List<File> result;

    // calculate matched files if
    // - we don't want to use cache (recalculate every time) OR
    // - directory was clearly updated after the last check OR
    // - last mtime change wasn't already checked for sure
    //   (system clock hasn't passed that second yet)
    if (!cachePatternMatching ||
        lastSeenParentDirMTime < currentParentDirMTime ||
        !(currentParentDirMTime < lastCheckedTime)) {
        //该参数 看源码自行配置
      lastMatchedFiles = sortByLastModifiedTime(getMatchingFilesNoCache(istraverse));
      lastSeenParentDirMTime = currentParentDirMTime;
      lastCheckedTime = now;
    }

    return lastMatchedFiles;
  }

测试 获取到多级目录监控文件