Specialized ExpressionPattern classes

mathics/core/definitions.py

@@ -10,7 +10,7 @@
 import pickle
 import re
 from collections import defaultdict
-from typing import Dict, List, Optional, Sequence, Set, Tuple, Union
+from typing import TYPE_CHECKING, Dict, List, Optional, Sequence, Set, Tuple, Union
 
 from mathics_scanner.tokeniser import full_names_pattern
 
@@ -29,6 +29,9 @@
 from mathics.core.util import canonic_filename
 from mathics.settings import ROOT_DIR
 
+if TYPE_CHECKING:
+    from mathics.core.evaluation import Evaluation
+
 # Collections of format symbols. Here we load some basic cases.
 # More symbols are populated from FormMeta classes (see `mathics.builtin.forms.base`)
 
@@ -82,6 +85,25 @@ def __init__(
             if not self.add_rule(rule):
                 print(f"{rule.pattern.expr} could not be associated with {self.name}")
 
+    def _resolve(self, evaluation: "Evaluation"):
+        """
+        Go over all the rules, and ensure that the corresponding patterns are in its final state
+        according to the current evaluation state.
+        """
+        for rule_list in (
+            self.ownvalues,
+            self.downvalues,
+            self.subvalues,
+            self.upvalues,
+            self.nvalues,
+            self.defaultvalues,
+        ):
+            for rule in rule_list:
+                rule._resolve(evaluation)
+        for rule_list in self.formatvalues.values():
+            for rule in rule_list:
+                rule._resolve(evaluation)
+
     def get_values_list(self, pos: str) -> List[BaseRule]:
         """Return one of the value lists"""
         assert pos.isalpha()
@@ -1105,6 +1127,7 @@ def load_builtin_definitions(
     """
     Load definitions from Builtin classes, autoload files and extension modules.
     """
+    from mathics.core.evaluation import Evaluation
     from mathics.core.load_builtin import (
         definition_contribute,
         mathics3_builtins_modules,
@@ -1132,5 +1155,11 @@ def load_builtin_definitions(
             with open(builtin_filename, "wb") as builtin_file:
                 pickle.dump(self.builtin, builtin_file, -1)
 
+    # Loop over definitions, to resolve on each rule
+    # which special kind of pattern must be considered.
+    evaluation = Evaluation(self)
+    for definition in self.builtin.values():
+        definition._resolve(evaluation)
+
     autoload_files(self, ROOT_DIR, "Autoload")
     autoload_files(self, osp.join(ROOT_DIR, "SystemFiles"), "Formats")

mathics/core/pattern/base.py

@@ -452,6 +452,7 @@ class ExpressionPattern(BasePattern):
     # get_pre_choices = pattern_nocython.get_pre_choices
     # match = pattern_nocython.match
     attributes: int
+    elements: Tuple[BasePattern, ...]
 
     def __init__(
         self,
@@ -463,10 +464,10 @@ def __init__(
         self.location = expr.location if hasattr(expr, "location") else None
         head = expr.head
         self.head = BasePattern.create(head, evaluation=evaluation)
-        self.elements = [
+        self.elements = tuple(
             BasePattern.create(element, evaluation=evaluation)
             for element in expr.elements
-        ]
+        )
 
     def _build_pattern_sort_key(self) -> tuple:
         return (
@@ -690,7 +691,7 @@ def get_match_count(self, vars_dict: Optional[dict] = None) -> Tuple[int, int]:
 
     def sort(self):
         """Sort the elements according to their sort key"""
-        self.elements.sort(key=lambda e: e.pattern_precedence)
+        self.elements = tuple(sorted(self.elements, key=lambda e: e.pattern_precedence))
 
 
 def expression_pattern_match_element_process_items(

mathics/core/pattern/common.py

@@ -15,13 +15,21 @@
 from mathics.core.expression import Expression
 from mathics.core.rules import is_option_rule
 from mathics.core.symbols import SymbolList
-from mathics.core.systemsymbols import SymbolDefault, SymbolOptional
+from mathics.core.systemsymbols import (
+    SymbolBlank,
+    SymbolBlankNullSequence,
+    SymbolBlankSequence,
+    SymbolDefault,
+    SymbolOptional,
+    SymbolPattern,
+)
 
 from .base import AtomPattern, BasePattern, ExpressionPattern
 
 
 def _is_option_like(candidate: BaseElement) -> bool:
     """Same shape check OptionsPattern.get_match_candidates() uses."""
+
     return is_option_rule(candidate) or candidate.has_form(SymbolList, None)
 
 
@@ -136,24 +144,149 @@ def match_expression_with_one_identity(
                 vars_dict[k] = old
 
 
+def _is_bare_blank(element: BaseElement) -> bool:
+    """
+    True for an unnamed Blank[] or Blank[Type] (0 or 1 sub-elements).
+    """
+    return element.has_form(SymbolBlank, 0, 1)
+
+
+def classify_fixed_blank_tuple(elements: tuple) -> Optional[tuple]:
+    """
+    If every entry of `elements` (zero or more of them) is either:
+      - a bare (unnamed) Blank[] / Blank[Type], or
+      - a named blank Pattern[name, Blank[...]] (`x_`, `x_Integer`, ...)
+    return `elements` back unchanged, as a signal that this
+    ExpressionPattern has a fixed arity with no possible backtracking
+    between slots: matching is exactly "N elements, checked
+    positionally, one does_match() per slot, in order", nothing else --
+    including the case of a name repeated across slots (`f[a_, a_]`),
+    since that's already handled *inside* Pattern.match's own
+    already-bound-name check (does_match on the second occurrence will
+    correctly require sameQ against the value bound by the first) --
+    this fast path doesn't need to special-case it.
+
+    Returns None (fall through to the general machinery) if any
+    element doesn't have this exact shape.
+    """
+    for element in elements:
+        if _is_bare_blank(element):
+            continue
+        if element.has_form(SymbolPattern, 2) and _is_bare_blank(element.elements[1]):
+            continue
+        return None
+    return elements
+
+
+def classify_single_sequence(elements: tuple) -> Optional[BaseElement]:
+    """
+    If `elements` is a 1-tuple whose sole entry is a NAMED
+    BlankSequence or BlankNullSequence -- `Pattern[name, BlankSequence[...]]`
+    (`s__`, `s__Integer`) or `Pattern[name, BlankNullSequence[...]]`
+    (`s___`, `s___Integer`) -- return that raw `Pattern[...]` element
+    unchanged, as a signal that this ExpressionPattern has exactly ONE
+    possible match: there is nothing else in the pattern to backtrack
+    against, so the whole expression's elements (however many there
+    are) are the only candidate for this single (Blank)(Null)Sequence.
+
+    BARE (unnamed) `__`/`___` are deliberately NOT included here, even
+    though structurally similar: as the sole element of a pattern with
+    no `rest_elements` after it, `less_first` is already `False` in
+    `_regular_match_element_sets`, so `subranges()` tries the
+    full-length split FIRST and succeeds immediately there -- no
+    combinatorial search actually happens for the bare case today, so
+    a dedicated class has nothing measurable to save (confirmed by
+    benchmarking: near-identical timings with/without this
+    classification for `head[__]`/`head[___]`, both typed and
+    untyped).
+
+    The NAMED case is different: profiling `head[s__Integer]` (see
+    session notes) against a matched Range[6000] showed the actual,
+    unavoidable per-element type check (`BlankSequence.match` itself)
+    takes about 15% of the total match time; the other ~85% is spent
+    BEFORE `match_element`'s `subranges()` call even gets to try
+    anything -- `basic_match_expression.yield_choice`'s pre-check via
+    `get_match_candidates_count`, followed by `match_element` building
+    `element_candidates` via `get_match_candidates` AGAIN (the exact
+    same O(n) type-scan, discarded immediately after: `subranges()` --
+    unlike `subsets()`, used by the Orderless path -- never even
+    consults its `included` argument, see its module docstring), plus
+    the `Expression(Sequence, *items)` (re)allocation in
+    `_yield_sequence_wrappings`. All of that exists to support
+    backtracking against OTHER elements/candidates that, for this
+    exact shape, don't exist: there is exactly one pattern element and
+    it must absorb the entire (possibly empty, possibly
+    single-element, possibly multi-element) sequence of expression
+    elements, or the match fails outright -- no split to search for.
+
+    Deliberately still excludes:
+    - Anything wrapped in Condition/Optional/PatternTest/Alternatives
+      around the (Blank)(Null)Sequence -- not exactly
+      `Pattern[name, Blank(Null)Sequence[...]]` shaped, conservatively
+      rejected (same policy as `classify_fixed_blank_tuple`).
+    - More than one element (`head[s__, t_]`, etc.) -- there IS
+      backtracking to do there (where does `s__`'s block end?), so
+      this fast path does not apply; that shape stays on the general
+      `subranges()`-based search.
+
+    Returns None (fall through to the general machinery) if `elements`
+    doesn't have exactly this shape.
+    """
+    if len(elements) != 1:
+        return None
+    element = elements[0]
+    if not element.has_form(SymbolPattern, 2):
+        return None
+    inner = element.elements[1]
+    if inner.has_form(SymbolBlankSequence, 0, 1) or inner.has_form(
+        SymbolBlankNullSequence, 0, 1
+    ):
+        return element
+    return None
+
+
+def match_fixed_blank_tuple(
+    blanks: tuple,
+    expr_elements: tuple,
+    vars_dict: dict,
+    evaluation: Evaluation,
+) -> Optional[dict]:
+    """
+    Positionally match `blanks[i]` against `expr_elements[i]` for every
+    i, threading vars_dict THROUGH EACH SLOT'S OWN yield_func -- same
+    protocol basic_match_expression/match_element use everywhere else
+    in this package -- rather than mutating a single shared dict in
+    place.
+
+    Returns the fully-threaded vars_dict on success (a dict that may or
+    may not be `is vars_dict`, depending on what each slot's match()
+    did), or None if any slot failed to match. Never mutates the
+    `vars_dict` object passed in.
+    """
+    current_vars = vars_dict
+    for blank, elt in zip(blanks, expr_elements):
+        result_box: list = []
+        blank.match(
+            elt,
+            {
+                "yield_func": lambda sub_vars, _rest, _box=result_box: _box.append(
+                    sub_vars
+                ),
+                "vars_dict": current_vars,
+                "evaluation": evaluation,
+                "fully": True,
+            },
+        )
+        if not result_box:
+            return None
+        current_vars = result_box[0]
+    return current_vars
+
+
 def _options_pattern_split(
     element: "BasePattern", rest_elements: tuple, candidates: tuple
 ):
     """
-    WMA quirk: when a pattern contains more than one OptionsPattern[]
-    element, only the LAST one ever collects any option-like arguments --
-    every earlier OptionsPattern[] always matches an empty sequence,
-    regardless of what option-like values are available. E.g.
-    `F[x_Integer, opt1:OptionsPattern[], opt2:OptionsPattern[]]` applied
-    to `F[z->p, 2, a->2, m->2]` binds opt1 to `{}` and opt2 to
-    `{a->2, m->2, z->p}` in WMA -- never split between the two.
-
-    The general subranges()/subsets()-based search doesn't know this: it
-    just finds *some* backtracking split between opt1 and opt2 (both have
-    unbounded, untyped match counts, so nothing else disambiguates which
-    split "is the one"), which usually isn't the WMA split and is wasted
-    search besides.
-
     Returns a `sets` list (same (items, (before, after)) shape the
     subranges()/subsets()-based paths produce) if `element` is an
     OptionsPattern[] (bare or Pattern-wrapped), or None if this fast path

mathics/core/pattern/deferred.py

@@ -7,13 +7,20 @@
 
 from typing import Optional
 
-from mathics.core.attributes import A_ORDERLESS
+from mathics.core.attributes import A_FLAT, A_ONE_IDENTITY, A_ORDERLESS
 from mathics.core.element import BaseElement
 from mathics.core.evaluation import Evaluation
 from mathics.core.expression import Expression
+from mathics.core.symbols import Symbol
 
 from .base import BasePattern, ExpressionPattern
-from .ordered import OrderedExpressionPattern
+from .common import classify_fixed_blank_tuple, classify_single_sequence
+from .ordered import (
+    FixedBlankTupleExpressionPattern,
+    OrderedExpressionPattern,
+    SimpleOrderedExpressionPattern,
+    SingleSequenceExpressionPattern,
+)
 from .orderless import OrderlessExpressionPattern
 
 
@@ -62,7 +69,7 @@ def __init__(self, expr: Expression):
         # pattern_precedence (see _build_pattern_sort_key below), which
         # is purely structural and must work even before resolution.
         self.head = BasePattern.create(expr.head)
-        self.elements = [BasePattern.create(element) for element in expr.elements]
+        self.elements = tuple(BasePattern.create(element) for element in expr.elements)
         self._impl: Optional[ExpressionPattern] = None
 
     def _resolve(self, evaluation: Evaluation) -> ExpressionPattern:
@@ -95,9 +102,31 @@ class that declares its own attributes statically in Python, or any
     Definitions is fully populated), use DeferredExpressionPattern
     instead.
     """
-    cls = (
-        OrderlessExpressionPattern
-        if A_ORDERLESS & attributes
-        else OrderedExpressionPattern
-    )
-    return cls(expr, attributes, evaluation)
+    if A_ORDERLESS & attributes:
+        return OrderlessExpressionPattern(expr, attributes, evaluation)
+    if (A_FLAT + A_ONE_IDENTITY) & attributes:
+        return OrderedExpressionPattern(expr, attributes, evaluation)
+    # No Orderless/Flat/OneIdentity: this is where the fixed-arity
+    # Blank-tuple shape (head[_], head[_,_], head[a_,b_], ...) gets
+    # decided -- BEFORE constructing any pattern objects, straight off
+    # the raw expr.elements (classify_fixed_blank_tuple is duck-typed
+    # to work on raw Expression children exactly like it does on
+    # BasePattern-wrapped ones). Doing the classification here, rather
+    # than as a runtime check inside a class's __init__/match, means
+    # neither SimpleOrderedExpressionPattern nor
+    # FixedBlankTupleExpressionPattern ever has to ask "wait, does this
+    # actually apply to me?" -- each is only ever constructed for the
+    # shape it's responsible for.
+    if isinstance(expr.head, Symbol):
+        if classify_fixed_blank_tuple(expr.elements) is not None:
+            return FixedBlankTupleExpressionPattern(expr, attributes, evaluation)
+        # head[s__], head[s__HEAD], head[s___], head[s___HEAD]: the
+        # single-named-Blank(Null)Sequence shape -- see
+        # classify_single_sequence's docstring in common.py for why
+        # this is a distinct, non-overlapping shape from the
+        # fixed-arity Blank tuple above (this one has exactly one
+        # element instead of two-or-more, and it's a Sequence, not a
+        # Blank).
+        if classify_single_sequence(expr.elements) is not None:
+            return SingleSequenceExpressionPattern(expr, attributes, evaluation)
+    return SimpleOrderedExpressionPattern(expr, attributes, evaluation)