Skip to content

Grow waste on the WPP2024 year-by-year population curve, not a frozen CAGR #42

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
HughRunyan wants to merge 1 commit into
base: main
Choose a base branch
from
+171 −43
Open

Grow waste on the WPP2024 year-by-year population curve, not a frozen CAGR #42

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 102 additions & 27 deletions SWEET_python/city_params.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,31 @@ def _build_oxidation_series(default_value, canonical_row, time_series_rows, year
# Cities can have multiple sets of CityParameters, one for each scenario.
# Sets of CityParameters can have one or more landfills, dumpsites, waste to energy, etc.
# Even for modeling a single landfill, City and CityParameters classes need to be used.
def _population_series_from_pop_data(pop_data, iso3, start_year=1990, end_year=2050):
"""Extract the WPP2024 per-year population Series for ``iso3`` from ``pop_data``.

``pop_data`` carries ``pop_1990``..``pop_2050`` columns when the yearly WPP
table is available (see helper_functions.load_population_data). Returns a
year-indexed Series, or ``None`` when the columns/country are absent so the
caller falls back to the frozen-CAGR ``growth_rate_*`` scalars.
"""
if pop_data is None or iso3 is None:
return None
try:
if iso3 not in pop_data.index:
return None
except Exception:
return None
years = list(range(start_year, end_year + 1))
cols = [f"pop_{y}" for y in years]
if not all(c in pop_data.columns for c in cols):
return None
vals = pd.to_numeric(pd.Series(pop_data.loc[iso3, cols].values), errors="coerce")
if vals.isna().all():
return None
return pd.Series(vals.values, index=years, dtype=float)


class CityParameters(BaseModel):
waste_fractions: Optional[Union[pd.DataFrame, pd.Series]] = None # WasteFractions
div_fractions: Optional[pd.DataFrame] = None # DiversionFractions
Expand All @@ -111,6 +136,9 @@ class CityParameters(BaseModel):
precip: Optional[float] = None
growth_rate_historic: Optional[float] = None
growth_rate_future: Optional[float] = None
# WPP2024 per-year population series (index = year), used to grow waste and
# diversion by P(year)/P(pivot). None -> fall back to the growth_rate_* CAGR.
population_series: Optional[pd.Series] = None
waste_per_capita: Optional[Union[pd.Series, float]] = None
precip_zone: Optional[str] = None
ks: Optional[DecompositionRates] = None
Expand Down Expand Up @@ -1670,6 +1698,8 @@ def sinar_city_and_site(self, row, linker, for_trace=False):
population = basics_dict["population"]
growth_rate_historic = basics_dict["growth_rate_historic"]
growth_rate_future = basics_dict["growth_rate_future"]
# None on the non-trace (wastemap/DST) path -> frozen-CAGR fallback.
population_series = basics_dict.get("population_series")
waste_mass = basics_dict["waste_mass"]
waste_per_capita = basics_dict["waste_per_capita"]
waste_fractions = basics_dict["waste_fractions"]
Expand Down Expand Up @@ -2114,6 +2144,8 @@ def site_only_estimate(self, row=None, pop_data=None):
population = basics_dict["population"]
growth_rate_historic = basics_dict["growth_rate_historic"]
growth_rate_future = basics_dict["growth_rate_future"]
# None on the non-trace (wastemap/DST) path -> frozen-CAGR fallback.
population_series = basics_dict.get("population_series")
waste_mass = basics_dict["waste_mass"]
waste_per_capita = basics_dict["waste_per_capita"]
waste_fractions = basics_dict["waste_fractions"]
Expand Down Expand Up @@ -2190,6 +2222,7 @@ def site_only_estimate(self, row=None, pop_data=None):
divs=divs,
defaults_used=defaults_used,
)
baseline.population_series = population_series
self.baseline_parameters = baseline
baseline._singapore_k(advanced_baseline=True)

Expand All @@ -2198,6 +2231,7 @@ def site_only_estimate(self, row=None, pop_data=None):
baseline.year_of_data_pop,
baseline.growth_rate_historic,
baseline.growth_rate_future,
baseline.population_series,
)

# Set up landfills
Expand Down Expand Up @@ -2336,6 +2370,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po
population = basics_dict.get("population", 100)
growth_rate_historic = basics_dict["growth_rate_historic"]
growth_rate_future = basics_dict["growth_rate_future"]
population_series = basics_dict.get("population_series")
waste_mass = basics_dict["waste_mass"]
waste_per_capita = basics_dict["waste_per_capita"]
waste_fractions = basics_dict["waste_fractions"]
Expand Down Expand Up @@ -2413,6 +2448,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po
divs=divs,
defaults_used=defaults_used,
)
baseline.population_series = population_series
self.baseline_parameters = baseline
baseline._singapore_k(advanced_baseline=True)

Expand All @@ -2421,6 +2457,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po
baseline.year_of_data_pop,
baseline.growth_rate_historic,
baseline.growth_rate_future,
baseline.population_series,
)

# Set up landfills
Expand Down Expand Up @@ -2636,6 +2673,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit
population = basics_dict.get("population", 100)
growth_rate_historic = basics_dict["growth_rate_historic"]
growth_rate_future = basics_dict["growth_rate_future"]
population_series = basics_dict.get("population_series")
waste_mass = basics_dict["waste_mass"]
waste_per_capita = basics_dict["waste_per_capita"]
waste_fractions = basics_dict["waste_fractions"]
Expand Down Expand Up @@ -2713,6 +2751,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit
divs=divs,
defaults_used=defaults_used,
)
baseline.population_series = population_series
self.baseline_parameters = baseline
baseline._singapore_k(advanced_baseline=True)

Expand All @@ -2721,6 +2760,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit
baseline.year_of_data_pop,
baseline.growth_rate_historic,
baseline.growth_rate_future,
baseline.population_series,
)

# Set up landfills
Expand Down Expand Up @@ -3239,7 +3279,15 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s
self.country = iso3s[iso3s['iso3'] == self.iso3]['name'].values[0]
self.region = defaults_2019.region_lookup[self.country]
population = 100


# WPP2024 per-year population series for this country (None -> the
# frozen-CAGR growth_rate_* scalars are used instead).
population_series = _population_series_from_pop_data(pop_data, self.iso3)
# True once waste_generated already carries population growth (the
# multi-row path bakes it into the projected series below), so the
# later WasteGeneratedDF.create must not apply growth a second time.
growth_already_applied = False

current_year = datetime.now().year
general_reference_year = current_year - 1

Expand Down Expand Up @@ -3305,22 +3353,29 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s

# Create time series from 1990 to 2050
years = np.arange(1990, 2051)
waste_series = np.zeros(len(years))

# Set 2020 as the baseline year
baseline_year = 2020
baseline_idx = baseline_year - 1990

# Project backward and forwards
for i, year in enumerate(years):
if year >= baseline_year:
# Future projection
years_since_baseline = year - baseline_year
waste_series[i] = avg_annual_waste * (growth_rate_future ** years_since_baseline)
else:
# Past projection - divide by historic growth rate (going backwards)
years_since_baseline = baseline_year - year
waste_series[i] = avg_annual_waste / (growth_rate_historic ** years_since_baseline)

# Anchor the projection at the data year (the latest observed waste
# year), the SAME pivot the single-row path uses -- so the two paths
# agree instead of differing by a 2020-vs-data-year offset. Fall back
# to 2020 only when the data year is unknown.
if pd.isna(year_of_data_msw):
baseline_year = 2020
else:
baseline_year = int(year_of_data_msw)

# Project the average waste around the baseline year by population
# growth. With a WPP series this is P(year)/P(baseline_year) -- the
# UN's decelerating trajectory; without one it is the frozen CAGR
# (growth_rate_future forward, growth_rate_historic backward), which
# growth_factors_for_years reproduces exactly around a single pivot.
growth_factors = growth_factors_for_years(
years,
baseline_year,
growth_rate_historic,
growth_rate_future,
population_series,
)
waste_series = avg_annual_waste * growth_factors

# Overwrite with actual data where available
for _, data_row in time_series_rows.iterrows():
Expand All @@ -3331,10 +3386,13 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s
if not np.isnan(val):
waste_series[year_idx] = val

# Store the time series
# Store the time series. This series already carries population
# growth, so the later WasteGeneratedDF.create must not re-apply it
# (previously it did -- a double application that inflated growth).
self.waste_time_series = pd.Series(waste_series, index=years)
waste_mass = self.waste_time_series
waste_per_capita = waste_mass * 1000 / population / 365
growth_already_applied = True

# Location data (canonical_row is typically a Series for TRACE usecase)
try:
Expand Down Expand Up @@ -3601,15 +3659,31 @@ def _is_transient_db_error(err: Exception) -> bool:
self.div_components["combustion"] = self.combustion_components
self.div_components["recycling"] = self.recycling_components

# Calculate waste generated, which is like waste masses but adjusts for population growth
waste_generated_df = WasteGeneratedDF.create(
waste_masses.loc[1990:2050, :],
1990,
2050,
year_of_data_pop,
growth_rate_historic,
growth_rate_future,
)
# Calculate waste generated, which is like waste masses but adjusts for
# population growth. The single-row path holds a flat base-year mass, so
# growth is applied here (WPP year-by-year, or the CAGR fallback). The
# multi-row path already baked growth into waste_masses above, so pass
# identity factors here to avoid the double application.
if growth_already_applied:
waste_generated_df = WasteGeneratedDF.create(
waste_masses.loc[1990:2050, :],
1990,
2050,
year_of_data_pop,
1.0,
1.0,
population_series=None,
)
else:
waste_generated_df = WasteGeneratedDF.create(
waste_masses.loc[1990:2050, :],
1990,
2050,
year_of_data_pop,
growth_rate_historic,
growth_rate_future,
population_series=population_series,
)

return {
"data_source_pop": 'UN',
Expand All @@ -3629,6 +3703,7 @@ def _is_transient_db_error(err: Exception) -> bool:
"temperature": temperature,
"waste_masses": waste_masses,
"waste_generated_df": waste_generated_df,
"population_series": population_series,
}
else:
data_source_waste = canonical_row['Data Source: Waste'].values[0]
Expand Down
85 changes: 69 additions & 16 deletions SWEET_python/class_defs.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,48 @@
from enum import Enum


def growth_factors_for_years(
years,
year_of_data_pop,
growth_rate_historic,
growth_rate_future,
population_series: Optional[pd.Series] = None,
):
"""Per-year multiplicative growth factors relative to ``year_of_data_pop``.

Two modes, selected by ``population_series``:

* **WPP year-by-year** (``population_series`` given): a per-year population
Series ``P(year)`` (UN WPP2024 medium variant). The factor is
``P(year) / P(year_of_data_pop)`` -- the UN's own decelerating trajectory,
with no extrapolation inside the modeling horizon. Any year missing from the
series is filled by forward/backward hold so the factor stays finite.

* **Frozen-CAGR fallback** (``population_series`` is ``None``): the legacy
behavior, ``growth_rate_historic ** t`` before the pivot and
``growth_rate_future ** t`` after it. Kept identical so the cities/DST
callers -- and any country absent from the yearly table -- are unchanged.

The pivot is passed in (not baked into a normalized series) so the same
population Series serves any pivot year, e.g. an ``implement_year`` scenario.
"""
years = np.asarray(years)
if population_series is not None and len(population_series) > 0:
P = population_series.reindex(years)
# Hold the nearest known value across any gap so the factor never goes NaN.
P = P.ffill().bfill()
pivot_val = population_series.reindex([year_of_data_pop]).ffill().bfill()
pivot = float(pivot_val.iloc[0])
if pivot and np.isfinite(pivot):
return (P.to_numpy(dtype=float) / pivot)
# Degenerate pivot -> fall through to the scalar path.
t = years - year_of_data_pop
growth_rate = np.where(
years < year_of_data_pop, growth_rate_historic, growth_rate_future
)
return np.power(growth_rate, t)


class WasteFractions(BaseModel):
food: float = 0.0
green: float = 0.0
Expand Down Expand Up @@ -155,15 +197,19 @@ def create(
year_of_data_pop: int,
growth_rate_historic: float,
growth_rate_future: float,
population_series: Optional[pd.Series] = None,
):
years = np.arange(start_year, end_year + 1)
t = years - year_of_data_pop

# Create growth rate array, using growth_rate_historic for years before year_of_data_pop and growth_rate_future after
growth_rate = np.where(
years < year_of_data_pop, growth_rate_historic, growth_rate_future
# WPP year-by-year factor when a population series is supplied; otherwise
# the frozen-CAGR fallback (see growth_factors_for_years).
growth_factors = growth_factors_for_years(
years,
year_of_data_pop,
growth_rate_historic,
growth_rate_future,
population_series,
)
growth_factors = growth_rate**t

# Apply growth factors to the waste_masses_df
adjusted_data = waste_masses_df.multiply(growth_factors, axis=0)
Expand Down Expand Up @@ -439,18 +485,23 @@ def create_advanced_baseline(
year_of_data_pop: int,
growth_rate_historic: float,
growth_rate_future: float,
population_series: Optional[pd.Series] = None,
):
compost = cls._apply_growth_rate(
divs.compost, year_of_data_pop, growth_rate_historic, growth_rate_future
divs.compost, year_of_data_pop, growth_rate_historic, growth_rate_future,
population_series,
)
anaerobic = cls._apply_growth_rate(
divs.anaerobic, year_of_data_pop, growth_rate_historic, growth_rate_future
divs.anaerobic, year_of_data_pop, growth_rate_historic, growth_rate_future,
population_series,
)
combustion = cls._apply_growth_rate(
divs.combustion, year_of_data_pop, growth_rate_historic, growth_rate_future
divs.combustion, year_of_data_pop, growth_rate_historic, growth_rate_future,
population_series,
)
recycling = cls._apply_growth_rate(
divs.recycling, year_of_data_pop, growth_rate_historic, growth_rate_future
divs.recycling, year_of_data_pop, growth_rate_historic, growth_rate_future,
population_series,
)

return cls(
Expand All @@ -466,15 +517,17 @@ def _apply_growth_rate(
year_of_data_pop: int,
growth_rate_historic: float,
growth_rate_future: float,
population_series: Optional[pd.Series] = None,
) -> pd.DataFrame:
years = df.index
t = years - year_of_data_pop

# Vectorized growth rate calculation
growth_rates = np.where(
years < year_of_data_pop, growth_rate_historic, growth_rate_future
# WPP year-by-year factor when a population series is supplied; otherwise
# the frozen-CAGR fallback (see growth_factors_for_years).
growth_factors = growth_factors_for_years(
df.index,
year_of_data_pop,
growth_rate_historic,
growth_rate_future,
population_series,
)
growth_factors = np.power(growth_rates, t)

# Apply growth factors across all waste types (columns) at once
adjusted_data = df.multiply(growth_factors, axis=0)
Expand Down