def Parse(url):
	try:
		parse_url = urlparse(url)
		base_url= '{uri.scheme}://{uri.netloc}'.format(uri=parse_url)
		html=get(url)
		link_re = re.compile(r'href="(.*?)"')
		allurls=link_re.findall(html)
		allurls =list(set(allurls))
		#print "Extracted URLs" + str(allurls)
		f_urls=[]
		f_urls.append(base_url)
		for ul in allurls:
			tmp_ul=''
			dblslah=False
			if ul.startswith("//"):
				dblslah=True
				if (urlparse(url).hostname) in (urlparse(ul).hostname):
					tmp_ul="http:"+ul
			elif ul.startswith("/") and dblslah==False:
				tmp_ul=base_url+ul
			elif ul.startswith("http"):
				if (urlparse(url).hostname) in (urlparse(ul).hostname):
					tmp_ul=ul
			if tmp_ul not in f_urls:
				f_urls.append(tmp_ul)

		f_urls =list(set(f_urls)) #Valid Unique URLs from Page
		#print "\n\nFormated URLS" + str(f_urls)
		#Extracting Path URLS
		final_urls=[]
		for u in f_urls:
			tmp_dbl_slash=u.replace("://","[X-0-0-X]",1)
			if ("://") not in tmp_dbl_slash: 
				if tmp_dbl_slash.endswith('/'):
					if tmp_dbl_slash not in final_urls:
						final_urls.append(tmp_dbl_slash.replace("[X-0-0-X]","://",1))
					tmp_dbl_slash=rreplace(tmp_dbl_slash,'/','',1)
				sep_count=tmp_dbl_slash.count("/")
				while(sep_count>0):
					x=tmp_dbl_slash.split('/')[-1]
					tmp_dbl_slash=rreplace(tmp_dbl_slash,x,"",1)
					if tmp_dbl_slash not in final_urls:
						final_urls.append(tmp_dbl_slash.replace("[X-0-0-X]","://",1))
					tmp_dbl_slash=tmp_dbl_slash[:-1]
					sep_count-=1
		final_urls=list(set(final_urls))
		return f_urls,final_urls
	except Exception as e:
		print "Error: " + str(e)
dataset/ETHPy150Open ajinabraham/WebAppSec/General Web Attacks/LFI/LFIcrawler.py/Parse/original