Add cross-property BM25 And operator

.github/workflows/main.yaml

@@ -29,7 +29,7 @@ env:
   WEAVIATE_135: 1.35.18
   WEAVIATE_136: 1.36.12
   WEAVIATE_137: 1.37.5-e0fe0d5.amd64
-  WEAVIATE_139: 1.39.0-rc.0-b41225e.amd64
+  WEAVIATE_139: 1.39.0-rc.1-89299a5.amd64
 
 jobs:
   lint-and-format:

integration/test_collection.py

@@ -36,6 +36,10 @@
 )
 from weaviate.collections.classes.internal import Object, ReferenceToMulti, _CrossReference
 from weaviate.collections.classes.types import PhoneNumber, WeaviateProperties, _PhoneNumber
+from weaviate.collections.grpc.shared import (
+    _BM25_AND_CROSS_MIN_VERSIONS,
+    _BM25_AND_CROSS_MIN_VERSIONS_STR,
+)
 from weaviate.exceptions import (
     UnexpectedStatusCodeError,
     WeaviateInsertInvalidPropertyError,
@@ -1756,3 +1760,24 @@ def test_bm25_operators(collection_factory: CollectionFactory) -> None:
     assert len(objs.objects) == 4
     assert objs.objects[0].uuid == uuid2
     assert sorted(obj.uuid for obj in objs.objects[1:]) == sorted([uuid1, uuid3, uuid4])
+
+
+def test_bm25_operator_and_cross(collection_factory: CollectionFactory) -> None:
+    collection = collection_factory(
+        properties=[
+            Property(name="title", data_type=DataType.TEXT),
+            Property(name="body", data_type=DataType.TEXT),
+        ],
+        vectorizer_config=Configure.Vectorizer.none(),
+    )
+
+    if not collection._connection._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+        pytest.skip(f"bm25 cross-property AND requires {_BM25_AND_CROSS_MIN_VERSIONS_STR}")
+
+    # Neither of `split_across`'s properties holds both tokens, so only cross-property AND matches it.
+    split_across = collection.data.insert({"title": "banana", "body": "split"})
+    single_property = collection.data.insert({"title": "banana split", "body": "dessert"})
+    collection.data.insert({"title": "banana", "body": "bread"})
+
+    objs = collection.query.bm25("banana split", operator=wvc.query.BM25Operator.and_cross())
+    assert sorted(obj.uuid for obj in objs.objects) == sorted([split_across, single_property])

test/collection/test_bm25_operator.py

@@ -0,0 +1,37 @@
+import pytest
+
+from weaviate.classes.query import BM25Operator
+from weaviate.collections.grpc.query import _QueryGRPC
+from weaviate.exceptions import WeaviateUnsupportedFeatureError
+from weaviate.proto.v1 import base_search_pb2
+from weaviate.util import _ServerVersion
+
+_AND_CROSS = base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS
+
+
+def _builder(version: str = "1.39.0") -> _QueryGRPC:
+    return _QueryGRPC(
+        weaviate_version=_ServerVersion.from_string(version),
+        name="Dummy",
+        tenant=None,
+        consistency_level=None,
+        validate_arguments=True,
+        uses_125_api=True,
+        uses_127_api=True,
+    )
+
+
+def test_and_cross_wired_into_request() -> None:
+    bm25 = _builder().bm25(query="banana split", operator=BM25Operator.and_cross())
+    assert bm25.bm25_search.search_operator.operator == _AND_CROSS
+
+    hybrid = _builder().hybrid(
+        query="banana split", alpha=0.0, bm25_operator=BM25Operator.and_cross()
+    )
+    assert hybrid.hybrid_search.bm25_search_operator.operator == _AND_CROSS
+
+
+@pytest.mark.parametrize("version", ["1.37.14", "1.38.7"])
+def test_and_cross_rejected_on_unsupported_versions(version: str) -> None:
+    with pytest.raises(WeaviateUnsupportedFeatureError):
+        _builder(version).bm25(query="banana split", operator=BM25Operator.and_cross())

test/test_server_version.py

@@ -60,6 +60,25 @@ def test_server_version_is_at_least(is_valid: bool) -> None:
     assert is_valid
 
 
+@pytest.mark.parametrize(
+    "version,expected",
+    [
+        ("1.36.9", False),
+        ("1.37.14", False),
+        ("1.37.15", True),
+        ("1.38.7", False),
+        ("1.38.8", True),
+        ("1.39.0", True),
+        ("1.40.0", True),
+    ],
+)
+def test_server_version_is_at_least_any(version: str, expected: bool) -> None:
+    assert (
+        _ServerVersion.from_string(version).is_at_least_any((1, 37, 15), (1, 38, 8), (1, 39, 0))
+        is expected
+    )
+
+
 def test_server_version_magic_methods() -> None:
     # Test __eq__
     assert _ServerVersion(1, 2, 3) == _ServerVersion(1, 2, 3)

weaviate/collections/classes/grpc.py

@@ -672,6 +672,13 @@ class BM25OperatorAnd(BM25OperatorOptions):
     operator = base_search_pb2.SearchOperatorOptions.OPERATOR_AND
 
 
+@dataclass
+class BM25OperatorAndCross(BM25OperatorOptions):
+    """Define the cross-property 'And' operator for keyword queries."""
+
+    operator = base_search_pb2.SearchOperatorOptions.OPERATOR_AND_CROSS
+
+
 class BM25OperatorFactory:
     """Define how the BM25 query's token matching should be performed."""
 
@@ -697,6 +704,20 @@ def and_() -> BM25OperatorOptions:
         """
         return BM25OperatorAnd()
 
+    @staticmethod
+    def and_cross() -> BM25OperatorOptions:
+        """Use the cross-property 'And' operator for keyword queries, where all query tokens must match across the searched properties combined.
+
+        Unlike `and_()`, which requires every token to occur within a single property, a token may be
+        matched by any of the searched properties, as long as each token is matched by at least one.
+
+        All searched properties must share the same tokenization and analyzer settings; the server
+        rejects the query otherwise.
+
+        Requires Weaviate `1.37.15`, `1.38.8`, `1.39.0` or higher.
+        """
+        return BM25OperatorAndCross()
+
 
 OneDimensionalVectorType = Sequence[NUMBER]
 """Represents a one-dimensional vector, e.g. one produced by the `Configure.Vectors.text2vec_jinaai()` module"""

weaviate/collections/grpc/query.py

@@ -24,7 +24,6 @@
     REFERENCE,
     REFERENCES,
     BM25OperatorOptions,
-    BM25OperatorOr,
     HybridFusion,
     HybridVectorType,
     Move,
@@ -243,14 +242,7 @@ def bm25(
                 base_search_pb2.BM25(
                     query=query,
                     properties=properties if properties is not None else [],
-                    search_operator=base_search_pb2.SearchOperatorOptions(
-                        operator=operator.operator,
-                        minimum_or_tokens_match=operator.minimum_should_match
-                        if isinstance(operator, BM25OperatorOr)
-                        else None,
-                    )
-                    if operator is not None
-                    else None,
+                    search_operator=self._bm25_operator_to_grpc(operator),
                 )
                 if query is not None
                 else None

weaviate/collections/grpc/shared.py

@@ -18,6 +18,7 @@
 from weaviate.collections.classes.config import ConsistencyLevel
 from weaviate.collections.classes.grpc import (
     MMR,
+    BM25OperatorAndCross,
     BM25OperatorOptions,
     BM25OperatorOr,
     HybridFusion,
@@ -35,6 +36,7 @@
 )
 from weaviate.exceptions import (
     WeaviateInvalidInputError,
+    WeaviateUnsupportedFeatureError,
 )
 from weaviate.proto.v1 import base_pb2, base_search_pb2
 from weaviate.types import NUMBER, UUID
@@ -49,6 +51,12 @@
 UINT32_LEN = 4
 UINT64_LEN = 8
 
+# Cross-property AND was backported to the 1.37 and 1.38 branches after landing in 1.39.
+_BM25_AND_CROSS_MIN_VERSIONS = ((1, 37, 15), (1, 38, 8), (1, 39, 0))
+_BM25_AND_CROSS_MIN_VERSIONS_STR = " or ".join(
+    f"{major}.{minor}.{patch}" for major, minor, patch in _BM25_AND_CROSS_MIN_VERSIONS
+)
+
 
 class _BaseGRPC:
     def __init__(
@@ -76,6 +84,30 @@ def _get_consistency_level(
             assert consistency_level.value == ConsistencyLevel.ALL
             return base_pb2.ConsistencyLevel.CONSISTENCY_LEVEL_ALL
 
+    def _bm25_operator_to_grpc(
+        self, bm25_operator: Optional[BM25OperatorOptions]
+    ) -> Optional["base_search_pb2.SearchOperatorOptions"]:
+        if bm25_operator is None:
+            return None
+
+        if isinstance(
+            bm25_operator, BM25OperatorAndCross
+        ) and not self._weaviate_version.is_at_least_any(*_BM25_AND_CROSS_MIN_VERSIONS):
+            raise WeaviateUnsupportedFeatureError(
+                "BM25Operator.and_cross()",
+                str(self._weaviate_version),
+                _BM25_AND_CROSS_MIN_VERSIONS_STR,
+            )
+
+        return base_search_pb2.SearchOperatorOptions(
+            operator=bm25_operator.operator,
+            minimum_or_tokens_match=(
+                bm25_operator.minimum_should_match
+                if isinstance(bm25_operator, BM25OperatorOr)
+                else None
+            ),
+        )
+
     def _recompute_target_vector_to_grpc(
         self,
         target_vector: Optional[TargetVectorJoinType],
@@ -741,14 +773,7 @@ def _parse_hybrid(
                 vector_distance=distance,
                 vectors=vectors,
                 selection=self._diversity_selection_to_grpc(diversity_selection),
-                bm25_search_operator=base_search_pb2.SearchOperatorOptions(
-                    operator=bm25_operator.operator,
-                    minimum_or_tokens_match=bm25_operator.minimum_should_match
-                    if isinstance(bm25_operator, BM25OperatorOr)
-                    else None,
-                )
-                if bm25_operator is not None
-                else None,
+                bm25_search_operator=self._bm25_operator_to_grpc(bm25_operator),
             )
             if query is not None or vector is not None
             else None

weaviate/outputs/query.py

@@ -9,6 +9,7 @@
 from weaviate.collections.classes.grpc import (
     MMR,
     BM25OperatorAnd,
+    BM25OperatorAndCross,
     BM25OperatorOr,
     ListOfVectorsQuery,
     NearVectorInputType,
@@ -62,6 +63,7 @@
     "GenerativeSearchReturnType",
     "GeoCoordinate",
     "BM25OperatorAnd",
+    "BM25OperatorAndCross",
     "BM25OperatorOr",
     "ListOfVectorsQuery",
     "MMR",

weaviate/proto/v1/v4216/v1/base_search_pb2.pyi

@@ -70,9 +70,11 @@ class SearchOperatorOptions(_message.Message):
         OPERATOR_UNSPECIFIED: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_OR: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_AND: _ClassVar[SearchOperatorOptions.Operator]
+        OPERATOR_AND_CROSS: _ClassVar[SearchOperatorOptions.Operator]
     OPERATOR_UNSPECIFIED: SearchOperatorOptions.Operator
     OPERATOR_OR: SearchOperatorOptions.Operator
     OPERATOR_AND: SearchOperatorOptions.Operator
+    OPERATOR_AND_CROSS: SearchOperatorOptions.Operator
     OPERATOR_FIELD_NUMBER: _ClassVar[int]
     MINIMUM_OR_TOKENS_MATCH_FIELD_NUMBER: _ClassVar[int]
     operator: SearchOperatorOptions.Operator

weaviate/proto/v1/v5261/v1/base_search_pb2.pyi

@@ -70,9 +70,11 @@ class SearchOperatorOptions(_message.Message):
         OPERATOR_UNSPECIFIED: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_OR: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_AND: _ClassVar[SearchOperatorOptions.Operator]
+        OPERATOR_AND_CROSS: _ClassVar[SearchOperatorOptions.Operator]
     OPERATOR_UNSPECIFIED: SearchOperatorOptions.Operator
     OPERATOR_OR: SearchOperatorOptions.Operator
     OPERATOR_AND: SearchOperatorOptions.Operator
+    OPERATOR_AND_CROSS: SearchOperatorOptions.Operator
     OPERATOR_FIELD_NUMBER: _ClassVar[int]
     MINIMUM_OR_TOKENS_MATCH_FIELD_NUMBER: _ClassVar[int]
     operator: SearchOperatorOptions.Operator

weaviate/proto/v1/v6300/v1/base_search_pb2.pyi

@@ -71,9 +71,11 @@ class SearchOperatorOptions(_message.Message):
         OPERATOR_UNSPECIFIED: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_OR: _ClassVar[SearchOperatorOptions.Operator]
         OPERATOR_AND: _ClassVar[SearchOperatorOptions.Operator]
+        OPERATOR_AND_CROSS: _ClassVar[SearchOperatorOptions.Operator]
     OPERATOR_UNSPECIFIED: SearchOperatorOptions.Operator
     OPERATOR_OR: SearchOperatorOptions.Operator
     OPERATOR_AND: SearchOperatorOptions.Operator
+    OPERATOR_AND_CROSS: SearchOperatorOptions.Operator
     OPERATOR_FIELD_NUMBER: _ClassVar[int]
     MINIMUM_OR_TOKENS_MATCH_FIELD_NUMBER: _ClassVar[int]
     operator: SearchOperatorOptions.Operator

weaviate/util.py

@@ -607,6 +607,17 @@ def from_string(cls, version: str) -> "_ServerVersion":
                 f"Unable to parse a version from the input string: {initial}. Is it in the format '(v)x.y.z' (e.g. 'v1.18.2' or '1.18.0')?"
             )
 
+    def is_at_least_any(self, *minimums: Tuple[int, int, int]) -> bool:
+        """Check a minimum that was backported to several release branches.
+
+        Each entry is the first version on its own minor branch to carry the feature, in ascending
+        order; the server is checked against the newest branch that is not newer than itself.
+        """
+        for major, minor, patch in reversed(minimums):
+            if (self.major, self.minor) >= (major, minor):
+                return self >= _ServerVersion(major, minor, patch)
+        return False
+
     def check_is_at_least_1_25_0(self, feature: str) -> None:
         if not self >= _ServerVersion(1, 25, 0):
             raise WeaviateUnsupportedFeatureError(feature, str(self), "1.25.0")