def filter_terms_by_df(doc_term_matrix, id_to_term,
                       max_df=1.0, min_df=1, max_n_terms=None):
    """
    Filter out terms that are too common and/or too rare (by document frequency),
    and compactify the top ``max_n_terms`` in the ``id_to_term`` mapping accordingly.
    Borrows heavily from the ``sklearn.feature_extraction.text`` module.

    Args:
        doc_term_matrix (:class:`scipy.sparse.csr_matrix <scipy.sparse.csr_matrix`):
            M X N matrix, where M is the # of docs and N is the # of unique terms
        id_to_term (dict): mapping of unique integer term identifiers to
            their corresponding normalized strings
        min_df (float in [0.0, 1.0] or int, optional): if float, value is the
            fractional proportion of the total number of documents and must be
            in [0.0, 1.0]; if int, value is the absolute number; filter terms
            whose document frequency is less than ``min_df``
        max_df (float in [0.0, 1.0] or int, optional): if float, value is the
            fractional proportion of the total number of documents and must be
            in [0.0, 1.0]; if int, value is the absolute number; filter terms
            whose document frequency is greater than ``max_df``
        max_n_terms (int, optional): only include terms whose *term* frequency
            is within the top `max_n_terms`

    Returns:
        :class:`scipy.sparse.csr_matrix <scipy.sparse.csr_matrix>`: sparse matrix
            of shape (# docs, # unique *filtered* terms), where value (i, j) is the
            weight of term j in doc i
        dict: id to term mapping, where keys are unique *filtered* integers as
            term ids and values are corresponding strings

    Raises:
        ValueError: if ``max_df`` or ``min_df`` or ``max_n_terms`` < 0
    """
    if max_df == 1.0 and min_df == 1 and max_n_terms is None:
        return doc_term_matrix, id_to_term
    if max_df < 0 or min_df < 0 or (max_n_terms is not None and max_n_terms < 0):
        raise ValueError('max_df, min_df, and max_n_terms may not be negative')

    n_docs, n_terms = doc_term_matrix.shape
    max_doc_count = max_df if isinstance(max_df, int) else int(max_df * n_docs)
    min_doc_count = min_df if isinstance(min_df, int) else int(min_df * n_docs)
    if max_doc_count < min_doc_count:
        raise ValueError('max_df corresponds to fewer documents than min_df')

    # calculate a mask based on document frequencies
    dfs = get_doc_freqs(doc_term_matrix, normalized=False)
    mask = np.ones(n_terms, dtype=bool)
    if max_doc_count < n_docs:
        mask &= dfs <= max_doc_count
    if min_doc_count > 1:
        mask &= dfs >= min_doc_count
    if max_n_terms is not None and mask.sum() > max_n_terms:
        tfs = get_term_freqs(doc_term_matrix, normalized=False)
        top_mask_inds = (tfs[mask]).argsort()[::-1][:max_n_terms]
        new_mask = np.zeros(n_terms, dtype=bool)
        new_mask[np.where(mask)[0][top_mask_inds]] = True
        mask = new_mask

    # map old term indices to new ones
    new_indices = np.cumsum(mask) - 1
    id_to_term = {new_indices[old_index]: term
                  for old_index, term in id_to_term.items()
                  if mask[old_index]}

    kept_indices = np.where(mask)[0]
    if len(kept_indices) == 0:
        msg = 'After filtering, no terms remain; try a lower `min_df` or higher `max_df`'
        raise ValueError(msg)

    return (doc_term_matrix[:, kept_indices], id_to_term)
dataset/ETHPy150Open chartbeat-labs/textacy/textacy/representations/vsm.py/filter_terms_by_df/original