src/curlient/crawler/ListCrawler.php
private function unique($pages) {
		$links = [];
		$pages = array_filter($pages, function ($page) use (&$links) {
			if (isset($links[ $page['url'] ])) {
				return false;
			}
			$links[ $page['url'] ] = 1;

			return true;
		});
		unset($links);

		return $pages;
	}
private function unique($pages) {
		$links = [];
		$pages = array_filter($pages, function ($page) use (&$links) {
			if (isset($links[ $page['url'] ])) {
				return false;
			}
			$links[ $page['url'] ] = 1;

			return true;
		});
		unset($links);

		return $pages;
	}
去重