-- Merging an already-replicated IVF segment into a larger one (tantivy
-- c4582807: IVF count() returns distinct docs, not posting rows). At the
-- prior rev, a replicated segment's posting-row total was read where a doc
-- count was meant, tripping debug-build accounting asserts in tantivy's IVF
-- plugin when such a segment became a merge SOURCE. pgrx regress builds are
-- debug builds, so this test exercises those asserts for real.
--
-- client_min_messages: the IVF merge emits a paradedb::ivf_build timings
-- NOTICE with nondeterministic millisecond values — keep it out of the
-- captured output.
SET client_min_messages = WARNING;
CREATE EXTENSION IF NOT EXISTS vector;
\i common/common_setup.sql

DROP TABLE IF EXISTS remerge;
CREATE TABLE remerge (
    id  int PRIMARY KEY,
    vec vector(16)
);

-- mutable_segment_rows = 0 routes every insert through immutable segments
-- (foreground merges only fire on an insert-cleanup that created a segment);
-- target_segment_count = 1 keeps the merge policy engaged (merging is
-- disabled while segment_count <= target); background_layer_sizes = '0'
-- keeps every merge in the foreground, deterministic. Inserts flush ~1000-doc
-- segments of ~70kb, and a 600kb layer closes its first candidate at
-- >= 10000 docs — at or above tantivy's vector_clustering_threshold, so the
-- merge target is written IVF (clustered), with every vector in 3 cells.
CREATE INDEX remerge_idx ON remerge
    USING bm25 (id, vec vector_l2_ops)
    WITH (
        key_field = id,
        cluster_replication = 3,
        target_segment_count = 1,
        mutable_segment_rows = 0,
        layer_sizes = '600kb',
        background_layer_sizes = '0'
    );

-- Wave 1: deterministic vectors (no random()), enough segments that the
-- 600kb candidate comfortably overfills even if per-segment bytes drift.
INSERT INTO remerge
SELECT g, ('[' || repeat((g % 89)::text || ',', 15) || (g % 89)::text || ']')::vector
FROM generate_series(1, 15000) g;

-- The merge produced a clustered segment...
SELECT bool_or(vector_format = 'ivf') AS has_ivf
FROM paradedb.vector_info('remerge_idx', 'vec');

-- ...whose reported vector count is distinct docs (= its doc count), not the
-- 3x posting-row total that replication writes...
SELECT bool_and(v.vector_num_vectors = i.num_docs) AS num_vectors_is_distinct_docs
FROM paradedb.vector_info('remerge_idx', 'vec') v
JOIN paradedb.index_info('remerge_idx') i USING (segno)
WHERE v.vector_format = 'ivf';

-- ...while the per-cluster sizes deliberately stay memberships: their total
-- strictly exceeds the distinct-doc count under replication.
SELECT sum(vector_total_memberships) > sum(vector_num_vectors)
         AS cluster_sizes_are_memberships
FROM paradedb.vector_info('remerge_idx', 'vec')
WHERE vector_format = 'ivf';

-- Wave 2: make the replicated IVF segment a merge SOURCE. The layer must
-- admit it (it is ~2.5-3.1mb; 3500kb leaves headroom) and the total corpus
-- must overfill the extended layer, which the extra 35000 rows guarantee.
ALTER INDEX remerge_idx SET (layer_sizes = '3500kb');
INSERT INTO remerge
SELECT g, ('[' || repeat((g % 89)::text || ',', 15) || (g % 89)::text || ']')::vector
FROM generate_series(15001, 50000) g;

SELECT bool_or(vector_format = 'ivf') AS still_has_ivf
FROM paradedb.vector_info('remerge_idx', 'vec');

SELECT bool_and(v.vector_num_vectors = i.num_docs) AS num_vectors_is_distinct_docs
FROM paradedb.vector_info('remerge_idx', 'vec') v
JOIN paradedb.index_info('remerge_idx') i USING (segno)
WHERE v.vector_format = 'ivf';

-- Exhaustive probing: the ceiling clamps to each segment's cluster count,
-- and LIMIT 60000 widens the candidate floor (4 x top_n) past the corpus so
-- the distance gate cannot stop early. The twice-merged replicated index
-- must return every distinct row exactly once — replicas deduped, nothing
-- lost, nothing doubled.
SET paradedb.vector_cluster_max_probe = 1.0;
SELECT count(*) AS returned, count(DISTINCT id) AS distinct_ids
FROM (
    SELECT id
    FROM remerge
    WHERE id @@@ pdb.all()
    ORDER BY vec <-> '[1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]'
    LIMIT 60000
) q;
RESET paradedb.vector_cluster_max_probe;

DROP TABLE remerge;
