MB-73803: Introducing Numeric Index Section

document/field_numeric_v2.go

@@ -0,0 +1,151 @@
+//  Copyright (c) 2026 Couchbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// 		http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package document
+
+import (
+	"fmt"
+	"reflect"
+
+	"github.com/blevesearch/bleve/v2/numeric"
+	"github.com/blevesearch/bleve/v2/numericv2"
+	"github.com/blevesearch/bleve/v2/size"
+	index "github.com/blevesearch/bleve_index_api"
+)
+
+var reflectStaticSizeNumericV2Field int
+
+func init() {
+	var f NumericV2Field
+	reflectStaticSizeNumericV2Field = int(reflect.TypeOf(f).Size())
+}
+
+// DefaultNumericV2IndexingOptions mirrors the v1 numeric defaults, minus
+// IncludeInAll
+const DefaultNumericV2IndexingOptions = index.StoreField | index.IndexField | index.DocValues
+
+// NumericV2Field is a numeric field indexed into the number_v2 section rather
+// than as prefix-coded terms in the inverted index.
+type NumericV2Field struct {
+	name              string
+	arrayPositions    []uint64
+	options           index.FieldIndexingOptions
+	value             numeric.PrefixCoded
+	numPlainTextBytes uint64
+}
+
+func (n *NumericV2Field) Size() int {
+	return reflectStaticSizeNumericV2Field + size.SizeOfPtr +
+		len(n.name) +
+		len(n.arrayPositions)*size.SizeOfUint64 +
+		len(n.value)
+}
+
+func (n *NumericV2Field) Name() string {
+	return n.name
+}
+
+func (n *NumericV2Field) ArrayPositions() []uint64 {
+	return n.arrayPositions
+}
+
+func (n *NumericV2Field) Options() index.FieldIndexingOptions {
+	return n.options
+}
+
+func (n *NumericV2Field) EncodedFieldType() byte {
+	return 'm'
+}
+
+// Analyze is a no-op: this field type is not tokenized. The number_v2 section
+// reads the value directly, and nothing about it needs analysis.
+func (n *NumericV2Field) Analyze() {
+}
+
+func (n *NumericV2Field) AnalyzedLength() int {
+	return 0
+}
+
+func (n *NumericV2Field) AnalyzedTokenFrequencies() index.TokenFrequencies {
+	return nil
+}
+
+// Value returns the prefix-coded, zero-shift representation, which is both the
+// stored-field form and the term written to this field's doc values.
+func (n *NumericV2Field) Value() []byte {
+	return n.value
+}
+
+// DocValue returns the term written to this field's doc values.
+// Duplicate needed to serve readers at the segment level
+func (n *NumericV2Field) DocValue() []byte {
+	return n.value
+}
+
+// SortableValue returns the value encoded as a uint64 whose unsigned ordering
+// matches the float64 ordering of the original number.
+func (n *NumericV2Field) SortableValue() uint64 {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0
+	}
+	return numericv2.EncodeInt64(i64)
+}
+
+func (n *NumericV2Field) Number() (float64, error) {
+	i64, err := n.value.Int64()
+	if err != nil {
+		return 0.0, err
+	}
+	return numeric.Int64ToFloat64(i64), nil
+}
+
+func (n *NumericV2Field) GoString() string {
+	return fmt.Sprintf("&document.NumericV2Field{Name:%s, Options: %s, Value: %s}",
+		n.name, n.options, n.value)
+}
+
+func (n *NumericV2Field) NumPlainTextBytes() uint64 {
+	return n.numPlainTextBytes
+}
+
+func NewNumericV2FieldFromBytes(name string, arrayPositions []uint64, value []byte) *NumericV2Field {
+	return &NumericV2Field{
+		name:              name,
+		arrayPositions:    arrayPositions,
+		value:             value,
+		options:           DefaultNumericV2IndexingOptions,
+		numPlainTextBytes: uint64(len(value)),
+	}
+}
+
+func NewNumericV2Field(name string, arrayPositions []uint64, number float64) *NumericV2Field {
+	return NewNumericV2FieldWithIndexingOptions(name, arrayPositions, number,
+		DefaultNumericV2IndexingOptions)
+}
+
+func NewNumericV2FieldWithIndexingOptions(name string, arrayPositions []uint64,
+	number float64, options index.FieldIndexingOptions) *NumericV2Field {
+	numberInt64 := numeric.Float64ToInt64(number)
+	prefixCoded := numeric.MustNewPrefixCodedInt64(numberInt64, 0)
+	return &NumericV2Field{
+		name:           name,
+		arrayPositions: arrayPositions,
+		value:          prefixCoded,
+		options:        options,
+		// not correct, just a place holder until we revisit how fields are
+		// represented and can fix this better
+		numPlainTextBytes: uint64(8),
+	}
+}

index/scorch/snapshot_index.go

@@ -28,6 +28,7 @@ import (
 	"github.com/RoaringBitmap/roaring/v2"
 	"github.com/blevesearch/bleve/v2/document"
 	geov2 "github.com/blevesearch/bleve/v2/geov2"
+	"github.com/blevesearch/bleve/v2/numericv2"
 	"github.com/blevesearch/bleve/v2/size"
 	"github.com/blevesearch/bleve/v2/util"
 	index "github.com/blevesearch/bleve_index_api"
@@ -53,6 +54,8 @@ type asynchSegmentResult struct {
 
 var reflectStaticSizeIndexSnapshot int
 var reflectStaticSizeIndexSnapshotGeoShapeV2Reader int
+var reflectStaticSizeIndexSnapshotNumericV2Reader int
+var reflectStaticSizeBitsetIterator int
 var reflectStaticSizeRoaringIntIterator int
 
 func init() {
@@ -69,6 +72,12 @@ func init() {
 	}
 	var gcr IndexSnapshotGeoShapeV2Reader
 	reflectStaticSizeIndexSnapshotGeoShapeV2Reader = int(reflect.TypeOf(gcr).Size())
+	var ncr IndexSnapshotNumericV2Reader
+	// via a pointer: the struct holds a sync.Once, and taking its type by value
+	// would copy a lock
+	reflectStaticSizeIndexSnapshotNumericV2Reader = int(reflect.TypeOf(&ncr).Elem().Size())
+	var bsi util.BitsetIterator
+	reflectStaticSizeBitsetIterator = int(reflect.TypeOf(bsi).Size())
 	var rip roaring.IntIterator
 	reflectStaticSizeRoaringIntIterator = int(reflect.TypeOf(rip).Size())
 }
@@ -556,6 +565,8 @@ func (is *IndexSnapshot) Document(id string) (rv index.Document, err error) {
 			rvd.AddField(document.NewGeoPointFieldFromBytes(name, arrayPos, value))
 		case 's':
 			rvd.AddField(document.NewGeoShapeFieldFromBytes(name, arrayPos, value))
+		case 'm':
+			rvd.AddField(document.NewNumericV2FieldFromBytes(name, arrayPos, value))
 		}
 
 		return true
@@ -1502,3 +1513,180 @@ func (g *IndexSnapshotGeoShapeV2Reader) Size() int {
 
 	return rv
 }
+
+func (i *IndexSnapshot) NumericV2FieldReader(ctx context.Context, field string) (
+	index.NumericV2FieldReader, error) {
+	rv := &IndexSnapshotNumericV2Reader{
+		field:     field,
+		hits:      make([]*util.Bitset, len(i.segment)),
+		iterators: make([]util.BitsetIterator, len(i.segment)),
+		snapshot:  i,
+	}
+
+	return rv, nil
+}
+
+type IndexSnapshotNumericV2Reader struct {
+	field string
+
+	// hits holds the per-segment result bitsets
+	hits      []*util.Bitset
+	iterators []util.BitsetIterator
+
+	// count is the total cardinality, computed on the first call and cached.
+	countOnce sync.Once
+	count     uint64
+
+	segmentOffset int
+
+	snapshot *IndexSnapshot
+}
+
+// Search evaluates the numeric range across all segments in the index snapshot.
+func (n *IndexSnapshotNumericV2Reader) Search(min, max *float64,
+	inclusiveMin, inclusiveMax *bool) error {
+
+	numSegments := len(n.snapshot.segment)
+	query := numericv2.NewRangeQuery(min, max, inclusiveMin, inclusiveMax)
+
+	var wg sync.WaitGroup
+	wg.Add(numSegments)
+
+	var errm sync.Mutex
+	var err error
+	// search each segment concurrently
+	for i := 0; i < numSegments; i++ {
+		go func(segID int) {
+			defer wg.Done()
+			err2 := n.searchSeg(segID, query)
+			if err2 != nil {
+				errm.Lock()
+				if err == nil {
+					err = err2
+				}
+				errm.Unlock()
+			}
+		}(i)
+	}
+	wg.Wait()
+
+	return err
+}
+
+// searchSeg evaluates the range against a single segment in the snapshot.
+func (n *IndexSnapshotNumericV2Reader) searchSeg(segID int,
+	query *numericv2.Query) error {
+
+	snapshot := n.snapshot.segment[segID]
+	numSeg, ok := snapshot.segment.(segment.NumericV2Segment)
+	if !ok {
+		return nil
+	}
+
+	data, err := numSeg.NumericV2Data(n.field)
+	if err != nil {
+		return err
+	}
+	if data == nil {
+		return nil
+	}
+	defer data.Close()
+
+	hits := query.Evaluate(data, snapshot.deleted, snapshot.segment.Count())
+
+	n.hits[segID] = hits
+	n.iterators[segID] = hits.Iterator()
+
+	return nil
+}
+
+// Next returns the next hit across all segments in the index snapshot.
+func (n *IndexSnapshotNumericV2Reader) Next(preAlloced *index.NumericV2FieldDoc) (
+	*index.NumericV2FieldDoc, error) {
+	rv := preAlloced
+	if rv == nil {
+		rv = &index.NumericV2FieldDoc{}
+	}
+
+	for n.segmentOffset < len(n.iterators) {
+		docNum, ok := n.iterators[n.segmentOffset].Next()
+		if !ok {
+			n.segmentOffset++
+			continue
+		}
+
+		rv.ID = index.NewIndexInternalID(rv.ID,
+			uint64(docNum)+n.snapshot.offsets[n.segmentOffset])
+		return rv, nil
+	}
+
+	return nil, nil
+}
+
+// Advance moves the reader to the first hit at or beyond the specified document.
+func (n *IndexSnapshotNumericV2Reader) Advance(ID index.IndexInternalID,
+	preAlloced *index.NumericV2FieldDoc) (*index.NumericV2FieldDoc, error) {
+	rv := preAlloced
+	if rv == nil {
+		rv = &index.NumericV2FieldDoc{}
+	}
+
+	num := ID.Value()
+
+	segIdx, localDocNum := n.snapshot.segmentIndexAndLocalDocNumFromGlobal(num)
+	if segIdx >= len(n.iterators) {
+		return nil, fmt.Errorf("error advancing to doc number %d, segment "+
+			"index %d out of bounds", num, segIdx)
+	}
+
+	if n.segmentOffset > segIdx {
+		return nil, fmt.Errorf("error advancing to doc number %d, segment "+
+			"index %d is less than current segment offset %d", num, segIdx, n.segmentOffset)
+	}
+
+	n.segmentOffset = segIdx
+	n.iterators[n.segmentOffset].AdvanceTo(int(localDocNum))
+
+	return n.Next(rv)
+}
+
+// Close drops the reader's per-segment results
+func (n *IndexSnapshotNumericV2Reader) Close() error {
+	for _, hits := range n.hits {
+		if hits != nil {
+			hits.Release()
+		}
+	}
+
+	n.hits = nil
+	n.iterators = nil
+	return nil
+}
+
+// Count returns the total number of matching documents across all segments.
+// The value is computed on the first call and cached for subsequent calls.
+func (n *IndexSnapshotNumericV2Reader) Count() uint64 {
+	n.countOnce.Do(func() {
+		for _, hits := range n.hits {
+			if hits != nil {
+				n.count += uint64(hits.Count())
+			}
+		}
+	})
+	return n.count
+}
+
+// Size returns the estimated size in bytes of the
+// IndexSnapshotNumericV2Reader, including its postings and iterators.
+func (n *IndexSnapshotNumericV2Reader) Size() int {
+	rv := reflectStaticSizeIndexSnapshotNumericV2Reader + size.SizeOfPtr +
+		len(n.field) + size.SizeOfInt
+
+	for _, hits := range n.hits {
+		rv += hits.SizeInBytes()
+	}
+
+	rv += reflectStaticSizeBitsetIterator * len(n.iterators)
+
+	return rv
+}