aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Util.java
static List<S3ObjectSummary> listObjectsChronologically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {

    //Algorithm:
    // - Full scan all objects that match the file name pattern and which are later than the file in the offset
    // - Select the oldest "fetchSize" number of files and return them.
    TreeSet<S3ObjectSummary> treeSet = new TreeSet<>((o1, o2) -> {
      int result = o1.getLastModified().compareTo(o2.getLastModified());
      if(result != 0) {
        //same modified time. Use name to sort
        return result;
      }
      return o1.getKey().compareTo(o2.getKey());
    });

    S3Objects s3ObjectSummaries = S3Objects
      .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.commonPrefix);

    // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call
    for (S3ObjectSummary s : s3ObjectSummaries) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        // remainingPrefix can be empty.
        // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
        // then the first objects returned here are:
        // myFolder/mySubFolder
        // myFolder/mySubFolder/myObject
        //
        // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
        // has no data
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix) && isEligible(s, s3Offset)) {
          treeSet.add(s);
        }
        if (treeSet.size() > fetchSize) {
          treeSet.pollLast();
        }
      }
    }

    return new ArrayList<>(treeSet);
  }
static List<S3ObjectSummary> listObjectsChronologically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {

    //Algorithm:
    // - Full scan all objects that match the file name pattern and which are later than the file in the offset
    // - Select the oldest "fetchSize" number of files and return them.
    TreeSet<S3ObjectSummary> treeSet = new TreeSet<>((o1, o2) -> {
      int result = o1.getLastModified().compareTo(o2.getLastModified());
      if(result != 0) {
        //same modified time. Use name to sort
        return result;
      }
      return o1.getKey().compareTo(o2.getKey());
    });

    S3Objects s3ObjectSummaries = S3Objects
      .withPrefix(s3Client, s3ConfigBean.s3Config.bucket, s3ConfigBean.s3Config.commonPrefix);

    // SDC-9413: since the s3ObjectSummaries is in lexical order, we should get all list of files in one api call
    for (S3ObjectSummary s : s3ObjectSummaries) {
      String fullPrefix = s.getKey();
      String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
      if (!remainingPrefix.isEmpty()) {
        // remainingPrefix can be empty.
        // If the user manually creates a prefix "myFolder/mySubFolder" in bucket "myBucket" and uploads "myObject",
        // then the first objects returned here are:
        // myFolder/mySubFolder
        // myFolder/mySubFolder/myObject
        //
        // All is good when pipeline is run but preview returns with no data. So we should ignore the empty file as it
        // has no data
        if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix) && isEligible(s, s3Offset)) {
          treeSet.add(s);
        }
        if (treeSet.size() > fetchSize) {
          treeSet.pollLast();
        }
      }
    }

    return new ArrayList<>(treeSet);
  }
Lists objects from AmazonS3 in chronological order [lexicographical order if 2 files have same timestamp] which are
later than or equal to the timestamp of the previous offset object

@param s3Client
@param s3ConfigBean
@param pathMatcher glob patterns to match file name against
@param s3Offset current offset which provides the timestamp of the previous object
@param fetchSize number of objects to fetch in one go
@return
@throws AmazonClientException