Source code for knowledgespaces.derivation.empirical

"""Descriptive structure construction by observed pattern frequency."""

from __future__ import annotations

import math
from collections.abc import Sequence
from numbers import Real

import numpy as np

from knowledgespaces.estimation.blim_em import ResponseMatrix
from knowledgespaces.metrics.frequencies import pattern_frequencies
from knowledgespaces.structures.knowledge_structure import KnowledgeStructure




def empirical_structure(
    data: ResponseMatrix | np.ndarray,
    *,
    threshold: float | None = None,
    items: Sequence[str] | None = None,
    counts: np.ndarray | None = None,
    max_memory_bytes: int = 512_000_000,
) -> KnowledgeStructure:
    """Select observed binary patterns with total weight >= threshold.

    The empty and full states are always included. ``None`` or zero uses
    ``ceil(N / 2**n_items)``, where N is the sum of observation weights.
    This implements the frequency-selection question of kstMatrix 3.0-0
    ``kmgenerate``. Its manual says "above"; the reference implementation
    and boundary probes use the inclusive comparison implemented here.

    Positive finite fractional thresholds and weights are supported. The
    default still takes the ceiling; it is not a proportion. Zero-weight
    rows cannot introduce states. Raw input columns default to item_1,
    item_2, ...; a ResponseMatrix supplies its own items and counts.

    Only observed patterns are counted: no power set or dense response
    table is allocated. This is a descriptive construction, not a latent
    BLIM estimate or a guarantee of union closure, learning-space axioms
    or statistical consistency. No response-error correction is applied.
    """
    if threshold is not None and (
        isinstance(threshold, (bool, np.bool_))
        or not isinstance(threshold, Real)
        or not math.isfinite(threshold)
        or threshold < 0
    ):
        raise ValueError("threshold must be finite and nonnegative, or None for automatic.")
    rows = data.patterns if isinstance(data, ResponseMatrix) else np.asarray(data)
    report = pattern_frequencies(
        data, n=None, items=items, counts=counts, max_memory_bytes=max_memory_bytes
    )
    if not np.isin(rows, [0, 1]).all():
        raise ValueError("Empirical structure generation requires binary responses.")
    if threshold is None or threshold == 0:
        # A positive ratio that underflows still has ceiling 1. Avoid
        # converting an enormous integer 2**n_items to a floating number.
        cutoff = max(1, math.ceil(math.ldexp(report.total_weight, -len(report.items))))
    else:
        cutoff = float(threshold)
    states = [
        frozenset(q for q, correct in zip(report.items, row, strict=True) if correct)
        for row, weight in zip(report.patterns, report.counts, strict=True)
        if weight >= cutoff
    ]
    return KnowledgeStructure(report.items, states)