def docs(self):
		for root, dirs, files in os.walk(self._root):
			for name in sorted(files):
				if name in self._invalids:
					continue

				try:
					elements = minidom.parse(os.path.join(root, name))
					for element in elements.getElementsByTagName('DOC'):
						doc = {}
						doc['id'] = element.getElementsByTagName('DOCID')[0].childNodes[0].data
						doc['issue'] = element.getElementsByTagName('ISSUE')[0].childNodes[0].data

						for cat in element.getElementsByTagName('CAT'):
							doc['categories_'+ cat.attributes['xml:lang'].value] = cat.childNodes[0].data.split('.')

						for date in element.getElementsByTagName('DATE'):
							if date.attributes['calender'].value == 'Persian':
								doc['date'] = date.childNodes[0].data

						elm = element.getElementsByTagName('TITLE')[0]
						doc['title'] = elm.childNodes[1].data if len(elm.childNodes) > 1 else ''

						doc['text'] = ''
						for item in element.getElementsByTagName('TEXT')[0].childNodes:
							if item.nodeType == 4:  # CDATA
								doc['text'] += item.data

						# refine text
						doc['text'] = self._paragraph_pattern.sub(r'\1\n', doc['text']).replace('\no ', '\n')

						yield doc

				except Exception as e:
					print('error in reading', name, e, file=sys.stderr)
dataset/ETHPy150Open sobhe/hazm/hazm/HamshahriReader.py/HamshahriReader.docs/original