aws-lib/src/main/java/com/streamsets/pipeline/stage/origin/s3/AmazonS3Util.java
static List<S3ObjectSummary> listObjectsLexicographically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {
    // Incrementally scan objects after the marker (s3Offset).
    List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
    listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
    listObjectsRequest.setMaxKeys(BATCH_SIZE);
    if (s3Offset.getKey() != null) {
      listObjectsRequest.setMarker(s3Offset.getKey());
    }

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    while (true) {
      for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
        String fullPrefix = s.getKey();
        String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
        if (!remainingPrefix.isEmpty()) {
          if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
            list.add(s);
          }
          // We've got enough objects.
          if (list.size() == fetchSize) {
            return list;
          }
        }
      }
      // Listing is complete. No more objects to be listed.
      if (!objectListing.isTruncated()) {
        break;
      }
      objectListing = s3Client.listNextBatchOfObjects(objectListing);
    }

    return list;
  }
static List<S3ObjectSummary> listObjectsLexicographically(
      AmazonS3 s3Client,
      S3ConfigBean s3ConfigBean,
      AntPathMatcher pathMatcher,
      S3Offset s3Offset,
      int fetchSize
  ) {
    // Incrementally scan objects after the marker (s3Offset).
    List<S3ObjectSummary> list = new ArrayList<>(fetchSize);

    ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
    listObjectsRequest.setBucketName(s3ConfigBean.s3Config.bucket);
    listObjectsRequest.setPrefix(s3ConfigBean.s3Config.commonPrefix);
    listObjectsRequest.setMaxKeys(BATCH_SIZE);
    if (s3Offset.getKey() != null) {
      listObjectsRequest.setMarker(s3Offset.getKey());
    }

    ObjectListing objectListing = s3Client.listObjects(listObjectsRequest);

    while (true) {
      for (S3ObjectSummary s : objectListing.getObjectSummaries()) {
        String fullPrefix = s.getKey();
        String remainingPrefix = fullPrefix.substring(s3ConfigBean.s3Config.commonPrefix.length(), fullPrefix.length());
        if (!remainingPrefix.isEmpty()) {
          if (pathMatcher.match(s3ConfigBean.s3FileConfig.prefixPattern, remainingPrefix)) {
            list.add(s);
          }
          // We've got enough objects.
          if (list.size() == fetchSize) {
            return list;
          }
        }
      }
      // Listing is complete. No more objects to be listed.
      if (!objectListing.isTruncated()) {
        break;
      }
      objectListing = s3Client.listNextBatchOfObjects(objectListing);
    }

    return list;
  }
Lists objects from AmazonS3 in lexicographical order

@param s3Client
@param s3ConfigBean
@param pathMatcher glob patterns to match file name against
@param s3Offset current offset which provides the key name of the previous object
@param fetchSize number of objects to fetch in one go
@return
@throws AmazonClientException