fix pyhton bug

.gitignore

@@ -53,3 +53,6 @@ python-package/.coverage
 /r-package/.Rhistory
 /r-package/.Rhistory
 .pdm-build
+
+# QGIS plugin build artifact (the repo must not contain zipped files)
+qgis-plugin/dist/

python-package/CHANGELOG.md

@@ -1,5 +1,32 @@
 # log history of geobr package development in Python
 
+-------------------------------------------------------
+# 1.0.1 version
+
+**Bug fixes**
+
+- Fixed `AttributeError: module 'pyarrow.compute' has no attribute
+  'match_substring_regex'`, which made **every** `read_*()` function fail on
+  pandas 3 when the installed pyarrow was built without RE2. Under pandas 3
+  strings are Arrow-backed, so a regex `str.contains()` dispatches to a pyarrow
+  kernel that such builds do not provide. Because the failure was in
+  `download_metadata_v2()`, it only appeared when the metadata cache had to be
+  rebuilt, so an existing `~/.cache/geobr` could mask it indefinitely. The
+  affected calls all match literal strings and now pass `regex=False`
+  (`utils.py`: `select_simplified()`, `download_metadata_v2()`, and the
+  `zone` filter in `select_metadata_v2()` used by `read_census_tract()`).
+  Found while running geobr inside QGIS 4.2.1, which ships pandas 3.0.3 and an
+  RE2-less pyarrow.
+
+- The download cache is now temporary, matching the behavior of the R
+  package: parquet and metadata files are stored in a session-specific
+  directory under the system temp folder and are removed when the Python
+  process exits. Previously, files persisted in `~/.cache/geobr` across
+  sessions, so data updated at the source was not picked up unless the user
+  cleared the cache manually. Cache directories left behind in
+  `~/.cache/geobr` by previous versions are no longer used and can be safely
+  deleted.
+
 -------------------------------------------------------
 # 1.0.0
 

python-package/geobr/_cache.py

@@ -1,26 +1,59 @@
-"""Disk-backed cache helpers for geobr parquet downloads."""
+"""Disk-backed cache helpers for geobr parquet downloads.
+
+Like the R package, the cache is temporary: files are stored in a
+session-specific directory under the system temp folder and are deleted when
+the Python process exits.
+"""
 
 from __future__ import annotations
 
-import os
+import atexit
+import shutil
+import tempfile
+import threading
+import time
 from pathlib import Path
 
+_cache_lock = threading.Lock()
+_session_dir: Path | None = None
+
+# Leftover session caches older than this are removed on a best-effort basis
+# when a new session cache is created (processes killed before atexit could
+# run leave their cache behind).
+_MAX_CACHE_AGE_DAYS = 30
+
 
 def cache_dir() -> Path:
-    """Return the geobr cache directory (~/.cache/geobr or temp fallback)."""
-    base = os.environ.get("XDG_CACHE_HOME")
-    if base:
-        path = Path(base) / "geobr"
-    else:
-        path = Path.home() / ".cache" / "geobr"
+    """Return the geobr cache directory for this session.
+
+    A fresh temporary directory is created for this session and deleted when
+    the Python process exits, so cached downloads never outlive the session
+    (same behavior as the R package).
+    """
+    global _session_dir
+
+    with _cache_lock:
+        if _session_dir is None:
+            _session_dir = Path(tempfile.mkdtemp(prefix="geobr_"))
+            _sweep_stale_caches()
+            atexit.register(_remove_session_dir, _session_dir)
+        return _session_dir
+
+
+def _sweep_stale_caches() -> None:
+    """Delete session caches abandoned by processes that did not exit cleanly."""
+    now = time.time()
+    max_age_seconds = _MAX_CACHE_AGE_DAYS * 24 * 60 * 60
     try:
-        path.mkdir(parents=True, exist_ok=True)
+        for entry in Path(tempfile.gettempdir()).glob("geobr_*"):
+            if entry.is_dir() and now - entry.stat().st_mtime > max_age_seconds:
+                shutil.rmtree(entry, ignore_errors=True)
     except OSError:
-        import tempfile
+        pass
+
 
-        path = Path(tempfile.gettempdir()) / "geobr"
-        path.mkdir(parents=True, exist_ok=True)
-    return path
+def _remove_session_dir(path: Path) -> None:
+    shutil.rmtree(path, ignore_errors=True)
 
 
 def cached_path(filename: str) -> Path:

python-package/geobr/utils.py

@@ -148,11 +148,17 @@ def select_simplified(metadata, simplified):
 
     """
 
+    # regex=False: literal match, and it avoids the missing RE2 kernel under
+    # pandas 3 with Arrow-backed strings (see download_metadata_v2).
     if simplified:
-        return metadata[metadata["download_path"].str.contains("simplified")]
+        return metadata[
+            metadata["download_path"].str.contains("simplified", regex=False)
+        ]
 
     else:
-        return metadata[~metadata["download_path"].str.contains("simplified")]
+        return metadata[
+            ~metadata["download_path"].str.contains("simplified", regex=False)
+        ]
 
 
 @lru_cache(maxsize=1240)
@@ -379,8 +385,15 @@ def download_metadata_v2() -> pd.DataFrame:
     temp_meta["year"] = pd.to_numeric(
         temp_meta["file_name"].str.extract(r"(\d+)", expand=False), errors="coerce"
     )
-    temp_meta["simplified"] = temp_meta["file_name"].str.contains(
-        "simplified", case=False, na=False
+    # Matched without a regex on purpose. Under pandas 3 strings are
+    # Arrow-backed, so a regex match dispatches to
+    # `pyarrow.compute.match_substring_regex`, which is missing from pyarrow
+    # builds compiled without RE2 - QGIS ships one, so `case=False` there
+    # raised AttributeError and no reader could resolve its metadata.
+    temp_meta["simplified"] = (
+        temp_meta["file_name"].str.lower().str.contains(
+            "simplified", na=False, regex=False
+        )
     )
     temp_meta.to_parquet(cache_meta, index=False)
     return temp_meta
@@ -415,7 +428,12 @@ def select_metadata_v2(geography, year, simplified=True, verbose=False, zone=Non
 
     # used for read_census_tract
     if zone:
-        temp_meta = temp_meta[temp_meta["file_name"].str.contains(zone)]
+        # regex=False for the same reason as download_metadata_v2: `zone` is a
+        # literal ("urban"/"rural"), and the regex path needs an RE2-enabled
+        # pyarrow that QGIS does not ship.
+        temp_meta = temp_meta[
+            temp_meta["file_name"].str.contains(zone, regex=False)
+        ]
 
     return temp_meta.iloc[0]
 

python-package/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "geobr"
-version = "1.0.0"
+version = "1.0.1"
 description = "geobr: Download Official Spatial Data Sets of Brazil"
 readme = "README.md"
 requires-python = "<4.0,>=3.10"

qgis-plugin/README.md

@@ -1,9 +1,13 @@
 # geobr for QGIS
 
-A QGIS plugin that exposes the [geobr](https://github.com/ipea/geobr) Python package as
-Processing algorithms. Every geobr reader — states, municipalities, census tracts, biomes,
-indigenous lands, health facilities, schools, favelas, polling places — becomes an algorithm in the
-Processing Toolbox.
+
+geobr is a computational package to download official spatial data sets
+of Brazil. The package covers a wide range of spatial data sets,
+available at various geographic scales and for various years with
+harmonized attributes, projection and fixed topology. This QGIS plugin 
+exposes the [geobr](https://github.com/ipea/geobr) Python package as
+Processing algorithms. geobr's readers — states, municipalities, census tracts, biomes,
+indigenous lands, health facilities, schools, favelas, polling places — become algorithms in the Processing Toolbox.
 
 Because they are Processing algorithms rather than a custom dialog, they work in **batch mode**, in
 **Model Builder**, and from the **`qgis_process`** command line, and they run on a background
@@ -96,10 +100,15 @@ the log, so an unexpected fallthrough is visible.
   from the Processing Toolbox and then quitting QGIS 4.2.1 was confirmed clean by the maintainer,
   so this is a `qgis_process` teardown problem only.
 - **QGIS 4.2.1 ships a geo stack outside geobr's declared bounds** — geopandas 1.1.4 (geobr pins
-  `<=1.1.2`), shapely 2.1.2 (pins `<=2.1.0`) and pandas 3.0.3. In testing, `read_state`,
-  `read_municipality` and `read_biomes` returned byte-identical feature counts to QGIS 3.42.1, so
-  it works today, but geobr does not claim support for these versions and a future geopandas or
-  pandas change could break it without warning.
+  `<=1.1.2`), shapely 2.1.2 (pins `<=2.1.0`) and pandas 3.0.3. This is not theoretical: pandas 3
+  makes strings Arrow-backed, so geobr's regex `str.contains()` dispatched to
+  `pyarrow.compute.match_substring_regex`, a kernel absent from QGIS's RE2-less pyarrow, and
+  **every reader failed** with `AttributeError`. It surfaced only when the metadata cache had to be
+  rebuilt, so a pre-existing `~/.cache/geobr` hid it. Fixed upstream in geobr by matching those
+  literal strings with `regex=False`; verified on QGIS 4.2.1 and 3.42.1 from a cold cache.
+  **This requires a geobr newer than 1.0.0** — on 1.0.0 the plugin works on QGIS 4 only while a
+  cache built elsewhere survives. The version bounds are still unclaimed territory, so treat other
+  pandas-3 breakage as possible.
 - **`read_health_region` is offered at municipality level only.** geobr accepts a `geometry_level`
   argument but ignores it: the `micro`/`macro` aggregation groups by every column it does not
   explicitly exclude, and `code_muni6` survives that `GROUP BY`, so all three levels return one
@@ -113,12 +122,25 @@ the log, so an unexpected fallthrough is visible.
   is worse than the cosmetic wart. Cast them in the field calculator if you need an integer join key.
 - **Downloads cannot be cancelled mid-request.** geobr fetches in one blocking call, so *Cancel*
   takes effect between stages, not during a transfer.
-- **`read_comparable_areas` uses geobr's legacy download path**, whose HTTP call has no timeout. On
-  a network that black-holes connections it can hang until QGIS is restarted. The other 30 readers
-  use the current parquet path.
-- **Stale cache.** geobr caches downloads in `~/.cache/geobr` with no expiry. If a dataset looks
-  out of date, delete that directory. The plugin does not expose a `cache` toggle, because geobr's
-  `cache=False` does not refresh the *metadata* — the thing that actually goes stale.
+- **`read_comparable_areas` is not exposed.** It is currently broken upstream, and it is also the
+  only reader still on geobr's legacy gpkg download path, whose `url_solver()` calls
+  `requests.get()` with **no timeout**. A Processing algorithm cannot be cancelled mid-request, so
+  on a network that black-holes connections that call would hang until QGIS is restarted. It is
+  excluded in `provider.py` (`_EXCLUDED_READERS`) and returns once geobr fixes it. Every other
+  reader uses the current parquet path.
+- **Downloads are cached per session.** As of geobr 1.0.1, the Python package stores downloads and
+  metadata in a fresh temp directory that is deleted when the Python process exits — the same
+  behavior as the R package. Source updates are picked up on the next QGIS start, with no action
+  needed. Within one session repeated reads are served from that session cache, but restarting QGIS
+  re-downloads whatever the session uses, and a single year of census tracts exceeds 350 MB.
+
+  geobr 1.0.0 instead cached persistently in `~/.cache/geobr` (or `$XDG_CACHE_HOME/geobr`), with no
+  expiry and no size cap, surviving restarts — so stale data went unnoticed unless the cache was
+  cleared by hand. Upgrading no longer uses that directory, but nothing removes it automatically
+  either; the **Clear geobr download cache** algorithm still deletes its contents. Tick *List files
+  only* on it first to see whether any legacy files remain. The plugin does not expose geobr's
+  `cache` argument, because `cache=False` does not refresh the *metadata* — the thing that actually
+  goes stale.
 - **Shapefile output truncates field names** to 10 characters. Prefer GeoPackage.
 
 ## Proxies
@@ -141,6 +163,7 @@ geobr_qgis/
 ├── __init__.py    classFactory, plugin lifecycle, dependency probe
 ├── provider.py    reader discovery (ast) + the Processing provider
 ├── algorithm.py   the one algorithm class that serves every reader
+├── cache.py       the "Clear geobr download cache" algorithm
 └── metadata.txt
 ```