def segment(self, use_preds=False, tokenize=False, output=None, list_only=False):
        """
        output all the text, split according to predictions or labels
        """
        sents = []
        thresh = 0.5
        sent = []
        frag = self.frag
        while frag:
            if tokenize: text = frag.tokenized
            else: text = frag.orig
            sent.append(text)
            if frag.ends_seg or (use_preds and frag.pred>thresh) or (not use_preds and frag.label>thresh):
                if not frag.orig: break
                sent_text = ' '.join(sent)
                if frag.ends_seg: spacer = '\n\n'
                else: spacer = '\n'
                if output: output.write(sent_text + spacer)
                elif not list_only: sys.stdout.write(sent_text + spacer)
                sents.append(sent_text)
                sent = []
            frag = frag.next
        return sents
dataset/ETHPy150Open daeilkim/refinery/refinery/lib/sbd.py/Doc.segment/original