Add indexed key-based retrieval and selective index build options

README.md

@@ -44,6 +44,40 @@ gtf-parser extract genes.gtf -t exon -f gene_id transcript_id exon_number -o exo
 gtf-parser extract genes.gtf -t exon -f gene_id --no-dedup
 ```
 
+### Build and use an index for faster repeated queries
+
+For large files and repeated lookups (for example repeatedly querying
+`-t gene`), build an index once and reuse it.
+
+```bash
+# Build index (prints output index path)
+gtf-parser index genes.gtf
+
+# Rebuild index after parser/index upgrades or source file changes
+gtf-parser index genes.gtf --force
+
+# Build a smaller/faster index for specific queries
+gtf-parser index genes.gtf -t gene -k gene_id gene_name
+
+# Use index for extraction
+gtf-parser extract genes.gtf -t gene -f gene_id gene_name --index genes.gtf.idx.sqlite
+
+# Use index for BED conversion
+gtf-parser bed genes.gtf gene -i gene_id gene_name --index genes.gtf.idx.sqlite
+
+# Retrieve one specific gene quickly by key
+gtf-parser extract genes.gtf -t gene -f gene_id gene_name --index genes.gtf.idx.sqlite --where gene_id=ENSG00000141510
+
+# Multiple accepted values for the same key
+gtf-parser extract genes.gtf -t gene -f gene_id gene_name --index genes.gtf.idx.sqlite --where gene_id=ENSG1,ENSG2
+
+# Combine filters (AND across keys)
+gtf-parser extract genes.gtf -t transcript -f transcript_id gene_id --index genes.gtf.idx.sqlite --where gene_id=ENSG1 --where transcript_id=ENST1
+```
+
+If an index is stale/incompatible or does not cover requested feature types/keys,
+the CLI prints a warning and falls back to plain file parsing.
+
 ### Convert to BED
 
 Convert features of a given type to BED6 format. The `name` column is built
@@ -71,12 +105,26 @@ The format is auto-detected from the first data line.
 ## Python API
 
 ```python
-from gtf_parser.parser import parse
+from gtf_parser.parser import build_index, parse
+
+idx = build_index("genes.gtf")
 
-for record in parse("genes.gtf", feature_types={"gene"}):
+for record in parse("genes.gtf", feature_types={"gene"}, index_path=idx):
     print(record.seqname, record.get("gene_id"), record.get("gene_name"))
 ```
 
+```python
+from gtf_parser.parser import parse
+
+for record in parse(
+    "genes.gtf",
+    feature_types={"gene"},
+    index_path="genes.gtf.idx.sqlite",
+    attribute_filters={"gene_id": {"ENSG00000141510"}},
+):
+    print(record.get("gene_id"), record.get("gene_name"))
+```
+
 ```python
 from gtf_parser.extract import extract
 
@@ -93,4 +141,4 @@ to_bed("genes.gtf", feature_type="gene", id_fields=["gene_id", "gene_name"])
 
 - [Edoardo Giacopuzzi](https://github.com/edg1983)
 
-With help from Claude Opus 4.6.
+With help from Claude Opus 4.6.

src/gtf_parser/bed.py

@@ -2,11 +2,12 @@
 
 from __future__ import annotations
 
+from collections.abc import Mapping, Set
 import sys
 from pathlib import Path
 from typing import TextIO
 
-from .parser import Record, parse
+from .parser import parse
 
 
 def to_bed(
@@ -16,6 +17,8 @@ def to_bed(
     id_separator: str = "|",
     output: TextIO | None = None,
     deduplicate: bool = True,
+    index_path: str | Path | None = None,
+    attribute_filters: Mapping[str, Set[str]] | None = None,
 ) -> None:
     """Convert GTF/GFF3 records of a given feature type to BED6 format.
 
@@ -40,11 +43,20 @@ def to_bed(
         Writable text stream (default: stdout).
     deduplicate:
         If ``True``, skip duplicate BED lines.
+    index_path:
+        Optional SQLite index path created with ``gtf-parser index``.
+    attribute_filters:
+        Optional attribute filters in the form ``{key: {value1, value2}}``.
     """
     out = output or sys.stdout
     seen: set[str] | None = set() if deduplicate else None
 
-    for record in parse(source, feature_types={feature_type}):
+    for record in parse(
+        source,
+        feature_types={feature_type},
+        index_path=index_path,
+        attribute_filters=attribute_filters,
+    ):
         name = id_separator.join(record.get(f) or "" for f in id_fields)
         # BED is 0-based half-open; GTF/GFF are 1-based inclusive
         bed_start = record.start - 1

src/gtf_parser/cli.py

@@ -29,18 +29,91 @@ def _add_common_args(sub: argparse.ArgumentParser) -> None:
         action="store_true",
         help="Disable deduplication of output rows",
     )
+    sub.add_argument(
+        "--index",
+        default=None,
+        help="Optional SQLite index path created with `gtf-parser index`",
+    )
+    sub.add_argument(
+        "--where",
+        action="append",
+        default=None,
+        help=(
+            "Attribute filter in the form key=value or key=v1,v2. "
+            "Can be repeated to combine filters."
+        ),
+    )
+
+
+def _parse_where_filters(where_filters: list[str] | None) -> dict[str, set[str]] | None:
+    if not where_filters:
+        return None
+
+    parsed: dict[str, set[str]] = {}
+    for raw_filter in where_filters:
+        key, sep, raw_values = raw_filter.partition("=")
+        if not sep:
+            raise ValueError(f"Invalid --where value '{raw_filter}': expected key=value")
+
+        clean_key = key.strip()
+        if not clean_key:
+            raise ValueError(f"Invalid --where value '{raw_filter}': empty key")
+
+        values = {value.strip() for value in raw_values.split(",") if value.strip()}
+        if not values:
+            raise ValueError(f"Invalid --where value '{raw_filter}': empty value list")
+
+        parsed.setdefault(clean_key, set()).update(values)
+
+    return parsed
+
+
+def _warn_if_index_unusable(
+    source: str,
+    index_path: str | None,
+    feature_types: set[str] | None = None,
+    attribute_filters: dict[str, set[str]] | None = None,
+) -> None:
+    if not index_path:
+        return
+
+    from .parser import index_status
+
+    usable, reason = index_status(
+        source,
+        index_path,
+        feature_types=feature_types,
+        attribute_filters=attribute_filters,
+    )
+    if not usable:
+        print(
+            f"Warning: index '{index_path}' will be ignored ({reason}).",
+            file=sys.stderr,
+        )
 
 
 def _run_extract(args: argparse.Namespace) -> None:
     from .extract import extract
 
+    try:
+        where_filters = _parse_where_filters(args.where)
+    except ValueError as exc:
+        raise SystemExit(str(exc))
+    feature_types = set(args.feature_types) if args.feature_types else None
+    _warn_if_index_unusable(
+        args.input,
+        args.index,
+        feature_types=feature_types,
+        attribute_filters=where_filters,
+    )
     output = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
     try:
-        feature_types = set(args.feature_types) if args.feature_types else None
         extract(
             source=args.input,
             fields=args.fields,
             feature_types=feature_types,
+            index_path=args.index,
+            attribute_filters=where_filters,
             output=output,
             separator=args.separator,
             deduplicate=not args.no_dedup,
@@ -53,13 +126,25 @@ def _run_extract(args: argparse.Namespace) -> None:
 def _run_bed(args: argparse.Namespace) -> None:
     from .bed import to_bed
 
+    try:
+        where_filters = _parse_where_filters(args.where)
+    except ValueError as exc:
+        raise SystemExit(str(exc))
+    _warn_if_index_unusable(
+        args.input,
+        args.index,
+        feature_types={args.feature_type},
+        attribute_filters=where_filters,
+    )
     output = open(args.output, "w", encoding="utf-8") if args.output else sys.stdout
     try:
         to_bed(
             source=args.input,
             feature_type=args.feature_type,
             id_fields=args.id_fields,
             id_separator=args.id_separator,
+            index_path=args.index,
+            attribute_filters=where_filters,
             output=output,
             deduplicate=not args.no_dedup,
         )
@@ -71,8 +156,13 @@ def _run_bed(args: argparse.Namespace) -> None:
 def _run_features(args: argparse.Namespace) -> None:
     from .parser import parse
 
+    try:
+        where_filters = _parse_where_filters(args.where)
+    except ValueError as exc:
+        raise SystemExit(str(exc))
+    _warn_if_index_unusable(args.input, args.index, attribute_filters=where_filters)
     features: set[str] = set()
-    for record in parse(args.input):
+    for record in parse(args.input, index_path=args.index, attribute_filters=where_filters):
         features.add(record.feature)
     for f in sorted(features):
         print(f)
@@ -81,14 +171,44 @@ def _run_features(args: argparse.Namespace) -> None:
 def _run_attributes(args: argparse.Namespace) -> None:
     from .parser import parse
 
+    try:
+        where_filters = _parse_where_filters(args.where)
+    except ValueError as exc:
+        raise SystemExit(str(exc))
     feature_types = set(args.feature_types) if args.feature_types else None
+    _warn_if_index_unusable(
+        args.input,
+        args.index,
+        feature_types=feature_types,
+        attribute_filters=where_filters,
+    )
     keys: set[str] = set()
-    for record in parse(args.input, feature_types=feature_types):
+    for record in parse(
+        args.input,
+        feature_types=feature_types,
+        index_path=args.index,
+        attribute_filters=where_filters,
+    ):
         keys.update(record.attributes.keys())
     for k in sorted(keys):
         print(k)
 
 
+def _run_index(args: argparse.Namespace) -> None:
+    from .parser import build_index
+
+    feature_types = set(args.feature_types) if args.feature_types else None
+    attribute_keys = set(args.attribute_keys) if args.attribute_keys else None
+    built = build_index(
+        source=args.input,
+        index_path=args.output,
+        force=args.force,
+        feature_types=feature_types,
+        attribute_keys=attribute_keys,
+    )
+    print(built)
+
+
 def main(argv: list[str] | None = None) -> None:
     parser = argparse.ArgumentParser(
         prog="gtf-parser",
@@ -154,6 +274,20 @@ def main(argv: list[str] | None = None) -> None:
         action="store_true",
         help="Disable deduplication of output rows",
     )
+    p_bed.add_argument(
+        "--index",
+        default=None,
+        help="Optional SQLite index path created with `gtf-parser index`",
+    )
+    p_bed.add_argument(
+        "--where",
+        action="append",
+        default=None,
+        help=(
+            "Attribute filter in the form key=value or key=v1,v2. "
+            "Can be repeated to combine filters."
+        ),
+    )
     p_bed.set_defaults(func=_run_bed)
 
     # --- features ---
@@ -162,6 +296,20 @@ def main(argv: list[str] | None = None) -> None:
         help="List all distinct feature types present in the file",
     )
     p_features.add_argument("input", help="Input GTF or GFF3 file")
+    p_features.add_argument(
+        "--index",
+        default=None,
+        help="Optional SQLite index path created with `gtf-parser index`",
+    )
+    p_features.add_argument(
+        "--where",
+        action="append",
+        default=None,
+        help=(
+            "Attribute filter in the form key=value or key=v1,v2. "
+            "Can be repeated to combine filters."
+        ),
+    )
     p_features.set_defaults(func=_run_features)
 
     # --- attributes ---
@@ -177,8 +325,55 @@ def main(argv: list[str] | None = None) -> None:
         default=None,
         help="Restrict to these feature type(s)",
     )
+    p_attrs.add_argument(
+        "--index",
+        default=None,
+        help="Optional SQLite index path created with `gtf-parser index`",
+    )
+    p_attrs.add_argument(
+        "--where",
+        action="append",
+        default=None,
+        help=(
+            "Attribute filter in the form key=value or key=v1,v2. "
+            "Can be repeated to combine filters."
+        ),
+    )
     p_attrs.set_defaults(func=_run_attributes)
 
+    # --- index ---
+    p_index = subs.add_parser(
+        "index",
+        help="Build an SQLite index for faster repeated retrieval",
+    )
+    p_index.add_argument("input", help="Input GTF or GFF3 file (plain or .gz)")
+    p_index.add_argument(
+        "-o",
+        "--output",
+        default=None,
+        help="Output index path (default: <input>.idx.sqlite)",
+    )
+    p_index.add_argument(
+        "--force",
+        action="store_true",
+        help="Overwrite existing index file if present",
+    )
+    p_index.add_argument(
+        "-t",
+        "--feature-types",
+        nargs="+",
+        default=None,
+        help="Only index these feature type(s) for faster/smaller indexes",
+    )
+    p_index.add_argument(
+        "-k",
+        "--attribute-keys",
+        nargs="+",
+        default=None,
+        help="Only index these attribute keys for --where lookups",
+    )
+    p_index.set_defaults(func=_run_index)
+
     args = parser.parse_args(argv)
     args.func(args)