diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..baf39f0 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,27 @@ + + +## What changed + +## Does this change a line marked `# decision:`? + +Those lines are Climate TRACE modelling choices, listed in `docs/MODEL_DECISIONS.md` in +RMI_Climate_TRACE_Waste_Methane. See "Decision comments" in the README. + +- [ ] No +- [ ] Yes. A Climate TRACE pull request on the same branch name updates the entry: + +## Do model numbers change? + +- [ ] No +- [ ] Yes. The PR is labelled `model-output-change`, and the changelog entry says how much they move. + +## Acceptance criteria + +- [ ] + +## Definition of done + +- Acceptance criteria met +- Tests pass +- Changelog updated +- Reviewed and merged diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..ac96429 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,316 @@ +name: Release + +# Cuts a SWEET release: an annotated tag sweet/YYYY.MM.N on a commit from main, and a +# GitHub Release for it whose notes list the pull requests merged since the previous +# release. Run it from the Actions tab ("Release" > "Run workflow"); with both inputs +# left at their defaults it releases the tip of main under the next free name for the +# current UTC month. The README's Releases section says what the name means and how a +# consumer asks for one. +# +# A release tag is never moved, reused or deleted -- an app pinned to it has to keep +# installing the same code -- so every check below runs before the tag exists, and the +# push is a plain one that git refuses if the name is already taken. +on: + workflow_dispatch: + inputs: + ref: + description: "What to release: a branch, tag or commit sha. It must be on main." + required: true + default: main + type: string + version: + description: "Release name, e.g. sweet/2026.10.0. Leave blank for the next free one this month (UTC)." + required: false + default: "" + type: string + +permissions: + contents: write # push the tag, create the GitHub Release + +# One release at a time, repository-wide, so two runs cannot both compute the same +# next number. A second run waits rather than cancelling the first. (GitHub keeps +# only one run waiting per group: a third click while one is running and one is +# waiting replaces the waiting one.) +concurrency: + group: release + cancel-in-progress: false + +jobs: + release: + name: Cut a release + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + # The first release's notes, written once: the Release step and the recovery + # command below must say the same thing. + FIRST_RELEASE_NOTE: "The first dated release of SWEET_python. What came before it is written up month by month in the changelog." + steps: + - name: Refuse to run release tooling from a branch + # "Use workflow from" picks which copy of this file runs. A branch's copy is + # unreviewed and could skip any check below, so only main's copy may tag. + # (Which commit gets released is the `ref` input, not this.) + if: github.ref != 'refs/heads/main' + run: | + echo "::error::Run the Release workflow from main (\"Use workflow from: main\"), not ${GITHUB_REF}. To release some other commit on main, put it in the 'ref' input." + exit 1 + + - name: Checkout + uses: actions/checkout@v7 + with: + # All history and all tags: reachability from main, the next free + # number and the previous release all need the whole picture. + fetch-depth: 0 + + - name: Decide what to release and what to call it + id: plan + env: + # Inputs reach the script as environment variables, never pasted into + # it, so a crafted ref cannot become shell. + REF: ${{ inputs.ref }} + VERSION_INPUT: ${{ inputs.version }} + run: | + set -euo pipefail + git fetch --force --tags origin + + REF="$(printf '%s' "$REF" | tr -d '[:space:]')" + if [[ -z "$REF" || "$REF" == -* ]]; then + echo "::error::'ref' must name a branch, tag or commit; got '${REF}'." + exit 1 + fi + + # A branch name means the branch on GitHub as of this run, then a tag, + # then a commit sha. Each lookup is spelled out so a tag and a branch + # sharing a name cannot be confused for each other. + if sha="$(git rev-parse --verify --quiet "refs/remotes/origin/${REF}^{commit}")"; then + kind=branch + elif sha="$(git rev-parse --verify --quiet "refs/tags/${REF}^{commit}")"; then + kind=tag + elif [[ "$REF" =~ ^[0-9a-fA-F]{7,40}$ ]] && sha="$(git rev-parse --verify --quiet "${REF}^{commit}")"; then + kind=commit + else + echo "::error::'${REF}' is not a branch, tag or commit in this repository." + exit 1 + fi + echo "'${REF}' is ${kind} -> commit ${sha}" + + # Releases come from main. A commit main does not contain is unreviewed + # (or not merged yet), and an app pinned to it would be pinned to code + # main may never have. + if ! git merge-base --is-ancestor "$sha" refs/remotes/origin/main; then + echo "::error::Commit ${sha} ('${REF}') is not on main. Releases are cut from main only: merge it first, then release main or a commit on it." + exit 1 + fi + + # A second release of the same commit would be a second name for code + # that already has one -- usually a double click on "Run workflow". + already="$(git tag --points-at "$sha" --list 'sweet/*' | python3 scripts/next_release.py --latest)" + if [[ -n "$already" ]]; then + echo "::error::Commit ${sha} is already released as ${already}. Use that release; there is nothing new to cut." + exit 1 + fi + + if [[ -z "$(printf '%s' "$VERSION_INPUT" | tr -d '[:space:]')" ]]; then + version="$(git tag --list 'sweet/*' | python3 scripts/next_release.py)" + echo "No version given; the next free name this month is ${version}." + else + # --check prints the name back trimmed, or explains what is wrong. + if ! version="$(python3 scripts/next_release.py --check "$VERSION_INPUT")"; then + echo "::error::'${VERSION_INPUT}' is not a release name. Use sweet/YYYY.MM.N, e.g. sweet/2026.10.0, or leave 'version' blank." + exit 1 + fi + fi + + # Never move or reuse a release: refuse a name that exists here or on + # GitHub. (The plain push below would also refuse it; this says why + # before the tests spend minutes.) + if git rev-parse --verify --quiet "refs/tags/${version}" >/dev/null \ + || git ls-remote --exit-code --tags origin "refs/tags/${version}" >/dev/null; then + echo "::error::${version} already exists. A release name is never moved or reused; leave 'version' blank to get the next free one." + exit 1 + fi + + # A later release must contain every earlier one, so that moving a pin to + # a newer name can never take a fix away. Main always satisfies this; + # only an old commit picked by sha or tag can fail it. + newest="$(git tag --list 'sweet/*' | python3 scripts/next_release.py --latest)" + if [[ -n "$newest" ]] && ! git merge-base --is-ancestor "refs/tags/${newest}" "$sha"; then + echo "::error::Commit ${sha} ('${REF}') does not contain ${newest}, the newest release. A release has to include the ones before it: release main, or a commit on main after ${newest}." + exit 1 + fi + # ...and its name has to sort after theirs, or "the newest release" + # stops meaning the one with the highest name (a hand-typed + # sweet/2026.09.9 after sweet/2026.10.0, say). + if [[ -n "$newest" ]] \ + && [[ "$(printf '%s\n%s\n' "$newest" "$version" | python3 scripts/next_release.py --latest)" != "$version" ]]; then + echo "::error::${version} would sort before ${newest}, the newest release. Pick a later name, or leave 'version' blank." + exit 1 + fi + + # The previous release is the newest one this commit contains -- after + # the check above, the newest release there is -- which is what "the pull + # requests since the last release" are counted from. Empty for the first. + previous="$(git tag --merged "$sha" --list 'sweet/*' | python3 scripts/next_release.py --latest)" + + # The month's hand-written changelog, linked by commit so the link shows + # the file as it was when released. A month nobody has written an entry + # for yet falls back to the index. + month="$(printf '%s' "$version" | sed -E 's#^sweet/([0-9]{4})\.([0-9]{2})\..*$#\1-\2#')" + base="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/blob/${sha}/changelog" + if git cat-file -e "${sha}:changelog/${month}.md" 2>/dev/null; then + changelog_line="The hand-written account of this month's changes, as of this release: [changelog/${month}.md](${base}/${month}.md)." + changelog_url="${base}/${month}.md" + else + changelog_line="There is no changelog/${month}.md at this commit yet; the monthly changelog files are listed in [changelog/README.md](${base}/README.md)." + changelog_url="${base}/README.md" + fi + + { + echo "ref=${REF}" + echo "sha=${sha}" + echo "kind=${kind}" + echo "version=${version}" + echo "previous=${previous}" + echo "changelog_line=${changelog_line}" + echo "changelog_url=${changelog_url}" + } >> "$GITHUB_OUTPUT" + + echo "Will release ${sha} as ${version} (previous release: ${previous:-none})." + + # Everything from here runs against the commit being released, so the tests + # are that commit's tests with that commit's pinned dependencies. The steps + # mirror .github/workflows/tests.yml; keep the two in step. + - name: Check out the commit being released + env: + SHA: ${{ steps.plan.outputs.sha }} + run: git checkout --quiet --detach "$SHA" + + - name: Setup Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + cache: pip + cache-dependency-path: requirements.txt + + - name: Install libpq headers + run: | + sudo apt-get update + sudo apt-get install -y libpq-dev + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install "pytest>=8,<10" + pip install --no-deps -e . + + - name: Run fast tests + # A failure here stops the release: nothing has been tagged yet. + run: pytest -m "not integration" --verbose + + - name: Tag the release + id: tag + env: + SHA: ${{ steps.plan.outputs.sha }} + VERSION: ${{ steps.plan.outputs.version }} + REF: ${{ steps.plan.outputs.ref }} + KIND: ${{ steps.plan.outputs.kind }} + run: | + set -euo pipefail + run_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" + # Annotated, so the tag records who cut it, when, and from what. The + # tagger is the bot that made it; the person who asked is in the message. + git -c user.name="github-actions[bot]" \ + -c user.email="41898283+github-actions[bot]@users.noreply.github.com" \ + tag --annotate --file=- "$VERSION" "$SHA" <> "$GITHUB_OUTPUT" + + - name: Summary + if: success() + env: + SHA: ${{ steps.plan.outputs.sha }} + VERSION: ${{ steps.plan.outputs.version }} + PREVIOUS: ${{ steps.plan.outputs.previous }} + CHANGELOG_URL: ${{ steps.plan.outputs.changelog_url }} + RELEASE_URL: ${{ steps.release.outputs.url }} + REF: ${{ steps.plan.outputs.ref }} + run: | + repo="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" + { + echo "## Released ${VERSION}" + echo + echo "| | |" + echo "|---|---|" + echo "| Tag | \`${VERSION}\` |" + echo "| Commit | [\`${SHA}\`](${repo}/commit/${SHA}) |" + echo "| Requested ref | \`${REF}\` |" + echo "| Previous release | ${PREVIOUS:-none (first release)} |" + echo "| Notes | [GitHub Release](${RELEASE_URL}) · [changelog](${CHANGELOG_URL}) |" + echo + echo "A consumer installs it with:" + echo + echo '```' + echo "pip install \"git+${repo}.git@${VERSION}\"" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Say how to finish if only the tag was made + # The tag is pushed and permanent, but the Release step failed. Running + # this workflow again would refuse the name (it exists now), so the + # Release has to be created by hand -- here is the exact command. + if: failure() && steps.tag.outcome == 'success' + env: + VERSION: ${{ steps.plan.outputs.version }} + PREVIOUS: ${{ steps.plan.outputs.previous }} + CHANGELOG_LINE: ${{ steps.plan.outputs.changelog_line }} + run: | + # The same notes the Release step would have written, changelog link included, + # each argument quoted for bash (`printf %q`) so the command pastes as is. + q() { printf '%q' "$1"; } + base="gh release create $(q "$VERSION") --repo $(q "$GITHUB_REPOSITORY") --verify-tag --title $(q "$VERSION")" + if [[ -n "$PREVIOUS" ]]; then + cmd="${base} --generate-notes --notes-start-tag $(q "$PREVIOUS") --notes $(q "$CHANGELOG_LINE")" + else + cmd="${base} --notes $(q "$(printf '%s\n\n%s' "$FIRST_RELEASE_NOTE" "$CHANGELOG_LINE")")" + fi + { + echo "## ${VERSION} is tagged, but has no GitHub Release" + echo + echo "The tag is pushed and stays. Do not delete or move it. Create the Release by hand:" + echo + echo '```' + echo "$cmd" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + echo "::error::${VERSION} is tagged but the GitHub Release was not created. See the job summary for the command that finishes it." diff --git a/.gitignore b/.gitignore index acd9af6..a6e35ba 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,5 @@ SWEET_python.egg-info /CLAUDE.md # Local PR-body drafts (edit in VS Code, sync to GitHub via `gh pr edit`) /pr-drafts/ +# Written into built copies by setup.py, never committed. +SWEET_python/_build_sha.py diff --git a/README.md b/README.md index 677eb01..da0eca0 100644 --- a/README.md +++ b/README.md @@ -46,47 +46,142 @@ that is the gap this workflow fills. Tests of this package's internals belong here, in the repo that owns them. # Usage -You will have to write your own code to import your data files. Examples are in SWEET_python/sweet_tools_obj.py—the load_from_database method illustrates the many different parameters that can be specified. For many parameters, default values are available. These are stored in the defaults_2019.py file, and the sweet_tools_obj.py file contains many examples of accessing them. The code for the model itself is in model.py. Models are generally run as part of a Landfill instance—the Landfill class is defined at the bottom of sweet_tools_obj.py -The standard way to run the model is shown by this code: +SWEET has no single entry point: each caller uses the path built for it. The two apps +that call it are the working examples — WasteMAP's API (`backend/sweet_launch.py` in +[RMI/WasteMAP](https://github.com/RMI/WasteMAP)) and the +[Climate TRACE waste-methane pipeline](https://github.com/RMI/RMI_Climate_TRACE_Waste_Methane). +Where things are: + +- **`SWEET_python/city_params.py`** — `City`, which sets up a city's waste, diversion + and landfills and runs them. Its main entry points: `load_csv_new` and + `load_andre_params` (a row of WasteMAP's cities table); `dst_baseline_blank` (a + Custom Location: a country, a population and the climate); + `cityparams_obj_for_blank_site` and `sdst_v1_5` (WasteMAP's single-site tool); and + `site_only_estimate_trace` (one Climate TRACE site). +- **`SWEET_python/landfill.py`** — `Landfill`: one disposal site (its type, open and + close years, gas capture and cover) and `estimate_emissions`, which runs the model + for it. +- **`SWEET_python/model_v2.py`** — `SWEET`, the first-order-decay engine. +- **`SWEET_python/advanced_dst.py`** and **`advanced_dst_city.py`** — the advanced DST, + where the caller supplies waste mass and composition as yearly series directly: + `run_advanced_dst` for one site, `run_advanced_dst_city` for a city and its + landfills. +- **`SWEET_python/defaults_2019.py`** — default parameters by country and region (waste + per person, composition, disposal split), alongside `mcf.py` (methane correction + factors), `population.py` (UN population, for growing waste year by year) and + `dst_common.py` (shared by the DST paths). + +# Decision comments + +Some lines carry a comment like `# decision: mcf-by-type-and-depth`. Each marks a modelling +choice that changes the numbers Climate TRACE publishes: a default value, a fallback, or a +simplification. The id names an entry in +[`docs/MODEL_DECISIONS.md`](https://github.com/RMI/RMI_Climate_TRACE_Waste_Methane/blob/main/docs/MODEL_DECISIONS.md) +in the Climate TRACE pipeline repo. The entry says what the choice is, why it was made, and how +sure we are of it. The list lives there, not here, because its ratings are about Climate TRACE's +published numbers. SWEET keeps no copy. + +Seven decisions are tagged in SWEET. + +- **Four are shared with WasteMAP's tools**, so changing one moves WasteMAP's numbers too: + - MCF, in `mcf.py` + - the guess for a site with no recorded type, in `site_type_defaults.py` + - waste composition, in `defaults_2019.waste_composition_for` + - population growth, in `class_defs.growth_factors_for_years` +- **Three are in the Climate TRACE paths** of `city_params.py`: gas capture, oxidation and + flaring. + +To change a tagged line: + +- Open a pull request in the Climate TRACE repo, from a branch with the same name as yours here, + that updates the entry. A replaced decision moves to that list's "Retired" section with the date. +- Keep the comment on the line it marks. Rename or remove it only when the entry changes too. + +The Climate TRACE pipeline's test checks these comments against its list. It reads them from the +SWEET release named in that repo's `sweet_python.ref`. A release that drops or renames a comment +therefore fails that repo's CI when the release is pinned, not this repo's. + +# Releases + +SWEET publishes dated releases. A release is an annotated git tag named +`sweet/YYYY.MM.N` on a commit from `main`, with a matching +[GitHub Release](https://github.com/RMI/SWEET_python/releases). + +## What the name means + +- **`sweet/`** — a release of this repository. The other tags here are other projects' + records of which SWEET commit they used: `wastemap/YYYY.MM.DD` (a wastemap.earth + deploy), `submission/MM_DD_YY` (a Climate TRACE submission) and the older + `v2026.08.17`. They are not SWEET releases, and the release workflow does not count + them. +- **`YYYY.MM`** — the year and month, in UTC, that the release was cut. Not the year of + any data or method in it. +- **`N`** — counts from 0 within the month: `sweet/2026.10.0`, `sweet/2026.10.1`, then + `sweet/2026.11.0`. + +A later name always contains everything in an earlier one: the workflow will not release +a commit that lacks the newest release, so moving a pin to a newer release can never take +a fix away. The name says nothing about how big the change is; the release notes and the +changelog do. + +## A release is never moved or reused + +Once `sweet/2026.10.0` exists, it points at the same commit for good. Nothing moves, +deletes or re-creates a release tag, because an app pinned to it has to keep installing +exactly the code it was tested with. A release with a mistake in it is fixed by merging +the fix to `main` and cutting the next release. + +## Cutting a release + +1. Merge what should be in it to `main`, with its changelog entry (see + [`changelog/README.md`](changelog/README.md)). +2. On GitHub, open **Actions → Release → Run workflow**, leave "Use workflow from" on + `main`, and run it. The defaults release the tip of `main` under the next free name + for this month. The two inputs are for when you need something else: + - **ref** — what to release: a branch, tag or commit sha. It has to be on `main`. + - **version** — a name of your own, e.g. `sweet/2026.10.3`. It has to be unused and + sort after the newest release. +3. Before anything is tagged, the workflow refuses a commit `main` does not contain, a + commit that is already a release, a commit that lacks the newest release, a name that + is taken, and a commit whose tests fail (the same `pytest -m "not integration"` as + CI, run against that commit). Then it tags the commit, pushes the tag and publishes + the GitHub Release, whose notes list the pull requests merged since the previous + release and link to that month's changelog file. The run's summary page shows the + tag, the commit and the links. + +If the tag gets pushed but the GitHub Release step fails, do not re-run the workflow: +the name is taken now, so it will refuse. The run's summary page gives the +`gh release create` command that finishes the job. + +## How an app asks for a release + +An app that uses SWEET names the release it wants in a small pin file in its own +repository (each app documents its own) and installs that tag: ``` -from SWEET_python.sweet_tools_obj import City -city = City(city_name) -city.load_from_database(db_file) # Replace this with your own data loading code -for landfill in city.non_zero_landfills: - landfill.estimate_emissions(baseline=True) - -city.organic_emissions_baseline = city.estimate_diversion_emissions(baseline=True) -city.total_emissions_baseline = city.sum_landfill_emissions(baseline=True) +pip install "git+https://github.com/RMI/SWEET_python.git@sweet/2026.10.0" ``` -The minimum information required to run the model is: -1) The country the city is in. This is used for some default parameters, and will look up a global region, used for other defaults. -2) Population or total waste generated per year. Default waste generation values are per capita, so city population is needed to calculate total waste generation if those data are not available. -3) Population growth rate, both historic (before the current year) and future (after current year). This is used to determine how waste generation changes backwards and forwards in time. Growth rates are percent growth rate + 1, e.g. 5% growth is represented by 1.05. Historic growth rates are above 1, not below, i.e. if population grew by an average of 5% per year from 1950 to the current date, the historic growth rate would be 1.05 not .95 -4) Mean annual precipitation in mm. Precipitation is used to determine waste decay rates. - -The SWEET model is designed to operate at the city level, and allows multiple landfills per city. If you would like to model only a single landfill, that can be done by creating only a single Landfill instance, directing all waste to it with the fraction_of_waste attribute, and setting all the diversion values to 0—no waste is composted, anaerobically digested, combusted/incinerated, or recycled, it all goes to the landfill. Country, population or waste generation, population growth rate, and precipitation are still required. - -## Release coordination - -SWEET_python is one of three repositories that build [wastemap.earth](https://wastemap.earth), -alongside [RMI/WasteMAP](https://github.com/RMI/WasteMAP) — which `pip`-installs this -package into its API and map-data pipeline — and the -[Climate TRACE waste-methane pipeline](https://github.com/RMI/RMI_Climate_TRACE_Waste_Methane). - -Production releases of wastemap.earth are coordinated by a shared `vYYYY.MM.DD` git -tag applied to the same release across all three repos. At each release, the commit of -SWEET_python that the WasteMAP API/pipeline images were built from is tagged with the -release version and recorded in WasteMAP's release manifest. This supersedes the old -date-named snapshot branches (`04_13_26`, `06_15_26`, …). - -> Note: SWEET_python is baked into two WasteMAP images with different timing — the batch -> map-data pipeline **and** the API, whose live decision-support endpoints (`/sdst`, -> `/adst`, `/cdst`) call this model at request time. A model change merged here reaches -> the live site only when WasteMAP builds a release against that commit (and, for the -> map, re-runs its pipeline). See the process doc below. - -See **[WasteMAP `RELEASES.md`](https://github.com/RMI/WasteMAP/blob/main/RELEASES.md)** -for the full release process. +A copy installed this way knows which commit it is: pip builds it from a git checkout, +and `setup.py` writes that commit into the package, so `SWEET_python.__version__` returns +it (`VERSION_SOURCE == "build"`). An editable install or a path dependency has no such +stamp; for those, the app sets `SWEET_GIT_SHA` to the commit it installed (both apps do), +and a local checkout falls back to reading its `.git`. Nothing reports a commit it +cannot vouch for: with none of the three, `__version__` is `"unknown"`. + +- On an app's **feature branch**, the pin may name a SWEET branch instead, to try a + model change before it is released. +- On an app's **`main` and production line**, the pin names a release or an exact + commit sha, never a branch. A branch moves: the next merge here would change what the + app runs without anyone changing the app. + +In a running app, `SWEET_python.__version__` is the commit it installed, not the release +name (the package docstring says why). To turn a commit back into a release name: +`git tag --points-at --list 'sweet/*'`. + +The pin matters most for WasteMAP's API, which calls SWEET at request time for its +decision-support tools (`/sdst`, `/adst`, `/cdst`): whatever SWEET its image was built +with is live the moment it deploys, even while the map's precomputed numbers came from an +older model. WasteMAP's +[`RELEASES.md`](https://github.com/RMI/WasteMAP/blob/main/RELEASES.md) covers its side. diff --git a/SWEET_python/__init__.py b/SWEET_python/__init__.py index c5b5cf6..d45c0cf 100644 --- a/SWEET_python/__init__.py +++ b/SWEET_python/__init__.py @@ -2,14 +2,17 @@ WHAT ``__version__`` IS, AND WHY IT IS A SHA -------------------------------------------- -It is the commit this copy was installed from -- not a hand-maintained number. - -SWEET never ships on its own. It reaches the world only inside a Climate TRACE run or a -WasteMAP deploy, and both of those already have identities of their own (a ``run_id`` and -a ``wastemap/YYYY.MM.DD`` tag). A ``MAJOR.MINOR`` maintained by hand here would be a -second, weaker name for a commit that is already recorded by sha in every run manifest, -and it would need bumping on every model change by the same person who has to remember -the tag. So there is no SWEET version number. There is a build stamp. +It is the commit this copy was installed from -- not a hand-maintained number, and not +the release name. + +SWEET's releases are git tags, ``sweet/YYYY.MM.N``, cut from ``main`` by +``.github/workflows/release.yml`` (the README's Releases section has the scheme). A +release names exactly one commit and is never moved, so a release's commit says as much +as its name -- and the commit is what an installed copy can report on its own, with no +repository and no network, and the only name a copy installed from a branch has. A copy +installed from ``sweet/2026.10.0`` reports that tag's commit; to turn a sha from a log +back into a release name, ``git tag --points-at --list 'sweet/*'``. So there is +still no version number in this file. There is a build stamp. WHAT IT IS FOR. The live DST endpoints (``/sdst``, ``/adst``, ``/cdst``) call this package at request time, which makes them the one place in the system where SWEET's @@ -20,16 +23,24 @@ HOW IT RESOLVES, in order, inventing nothing: -1. ``SWEET_GIT_SHA``, which the image build sets from the ref it actually installed. -2. A ``.git`` directory beside the package, for an editable checkout. -3. ``"unknown"`` -- an honest answer, and the one a reader must be able to distinguish - from a real sha. ``VERSION_SOURCE`` says which of the three it was. +1. The build stamp, ``SWEET_python/_build_sha.py``: ``setup.py`` writes the commit into + every copy built from a clean git checkout -- which is how ``pip install + git+https://...@sweet/2026.10.0`` builds -- so the copy itself knows. First because + it describes the installed code, where an environment variable is a claim made from + outside it. +2. ``SWEET_GIT_SHA``, which the apps set from the commit they install, for a copy with + no stamp (an editable install, or a path dependency). +3. A ``.git`` directory beside the package, for an editable checkout. +4. ``"unknown"`` -- an honest answer, and the one a reader must be able to distinguish + from a real sha. ``VERSION_SOURCE`` says which of the four it was: ``build``, + ``env``, ``git`` or ``unavailable``. Same shape, deliberately, as the Climate TRACE pipeline's ``results_tables.resolve_git_shas``: env first, then a repository read, then a recorded absence. -See ``VERSIONING.md`` in RMI_Climate_TRACE_Waste_Methane for the whole scheme. +``VERSIONING.md`` in RMI_Climate_TRACE_Waste_Methane covers the identifiers around this +one: a run's ``run_id`` and the ``wastemap/`` and ``submission/`` tags. """ from __future__ import annotations @@ -58,7 +69,18 @@ def _sha_from_git(repo_dir: Path) -> Optional[str]: return (out.stdout or "").strip() or None +def _sha_from_build() -> Optional[str]: + try: + from SWEET_python._build_sha import GIT_SHA + except ImportError: + return None + return (GIT_SHA or "").strip() or None + + def _resolve_version() -> "tuple[str, str]": + sha = _sha_from_build() + if sha: + return sha, "build" sha = (os.environ.get("SWEET_GIT_SHA") or "").strip() if sha: return sha, "env" diff --git a/SWEET_python/city_params.py b/SWEET_python/city_params.py index a7b7e8f..5f91894 100644 --- a/SWEET_python/city_params.py +++ b/SWEET_python/city_params.py @@ -1,5 +1,7 @@ import os +import re import sys +import unicodedata from pathlib import Path sys.path.append("/app/SWEET_python/SWEET_python") @@ -23,6 +25,7 @@ from SWEET_python.singapore_k import compute_singapore_k import SWEET_python.defaults_2019 as defaults_2019 import SWEET_python.mcf as mcf_defaults +import SWEET_python.site_type_defaults as site_type_defaults from SWEET_python.population import average_growth_rates, country_population_series import psycopg2 from psycopg2.extras import RealDictCursor @@ -38,6 +41,43 @@ ) +#: Column the Climate TRACE pipeline adds to a site's rows when they come from the +#: consolidated facility table: the year each row describes. Its presence is what marks +#: rows as one per YEAR of a site, rather than the legacy input's one per SOURCE record. +ROW_YEAR_COLUMN = "row_year" + + +def _carried_series(years, values, years_range): + """One value per model year from the years a site reported one, or None if none did. + + One distinct value is that value in every year. Several make a step series: the + first reported value is carried back to the start of ``years_range``, each value + forward until the next report, and the latest forward to the end. Two values in one + year are averaged first. + + Used for rows that carry their own year (``ROW_YEAR_COLUMN``). The consolidated + facility table resolves gas collection efficiency and oxidation once per site today + and repeats the value on every year, so this gives a constant; if upstream starts + resolving them per year, the series follows the reports with no change here. + """ + # na_value: a nullable column (Int64, Float64) holds pd.NA, which a plain float + # conversion refuses. + years = pd.to_numeric(pd.Series(years, dtype=object), errors="coerce").to_numpy(dtype=float, na_value=np.nan) + values = pd.to_numeric(pd.Series(values, dtype=object), errors="coerce").to_numpy(dtype=float, na_value=np.nan) + usable = ~np.isnan(years) & ~np.isnan(values) + if not usable.any(): + return None + reported = ( + pd.Series(values[usable], index=years[usable].astype(int)) + .groupby(level=0).mean().sort_index() + ) + index = pd.Index(years_range) + if reported.nunique() == 1: + return pd.Series(float(reported.iloc[0]), index=index) + full = reported.reindex(reported.index.union(index)).ffill().bfill() + return full.reindex(index).astype(float) + + def _build_oxidation_series(default_value, canonical_row, time_series_rows, years_range): """Per-year oxidation factor for one modeled landfill, preferring per-site input oxidation over the type/gas-capture default. @@ -57,6 +97,7 @@ def _build_oxidation_series(default_value, canonical_row, time_series_rows, year sites; ``canonical_row`` is the single deduped row. Either may carry ``oxidation``. """ years_index = pd.Index(years_range) + # decision: oxidation series = pd.Series(float(default_value), index=years_index) # Assemble the site's input rows as a frame so single-row (Series) and multi-row @@ -89,6 +130,17 @@ def _build_oxidation_series(default_value, canonical_row, time_series_rows, year if not ox.notna().any(): return series # no input oxidation -> keep the type/gas-capture default + # Rows that carry their own year (the consolidated input): each value belongs to + # that year, so carry the reports back and forward rather than averaging them. + if ROW_YEAR_COLUMN in frame.columns: + carried = _carried_series(frame[ROW_YEAR_COLUMN], ox, years_index) + if carried is None: + # Values, but no row with a usable year: still the site's own value, in every + # year. The emissions-year rule below is the legacy input's, and does not get + # to decide these rows. + carried = pd.Series(float(ox[ox.notna()].mean()), index=years_index) + return carried + # 1) Site-wide mean as the full-series baseline. series[:] = float(ox[ox.notna()].mean()) @@ -161,6 +213,7 @@ def _resolve_gas_capture(flag, site_type, canonical_row, time_series_rows, years except Exception: presence = False + # decision: oxidation oxidation_value = OX_OPTIONS["ox_cap" if presence else "ox_nocap"][site_type] # Measured per-year capture, where the site has it. Keyed on @@ -177,6 +230,26 @@ def _resolve_gas_capture(flag, site_type, canonical_row, time_series_rows, years # such a site reaches this branch rather than the reported pathway. Taking .mean() # of the emptied frame yielded NaN and poisoned the whole capture series; fall back # to the gated default instead, matching the scalar path below. + # Rows that carry their own year (the consolidated input): the value belongs to + # that year, whether or not the site reported CH4 in it. Keyed on the emissions + # year below, a modelled site -- which has none -- lost its measured efficiency to + # the type default. + if ( + isinstance(time_series_rows, pd.DataFrame) + and ROW_YEAR_COLUMN in time_series_rows.columns + and 'gas_collection_efficiency' in time_series_rows.columns + ): + efficiency = pd.to_numeric(time_series_rows['gas_collection_efficiency'], errors='coerce') + carried = _carried_series(time_series_rows[ROW_YEAR_COLUMN], efficiency, years_range) + if carried is None and efficiency.notna().any(): + # A measured value on rows with no usable year is still the site's own value. + # Falling through to the emissions-year rule below would drop it for the + # default, which is the fault this branch exists to fix. + carried = pd.Series(float(efficiency[efficiency.notna()].mean()), index=years_range) + if carried is not None: + return presence, oxidation_value, carried + # No measured value at all: the gated default below, as for any site. + gascap_df = None if isinstance(time_series_rows, pd.DataFrame): if time_series_rows['gas_collection_efficiency'].notna().any(): @@ -200,6 +273,7 @@ def _resolve_gas_capture(flag, site_type, canonical_row, time_series_rows, years time_series_rows, pd.DataFrame ) else np.nan if pd.isna(value): + # decision: gas-capture-efficiency value = GAS_EFF_OPTIONS[site_type] if presence else 0 return presence, oxidation_value, pd.Series(value, index=years_range) @@ -237,16 +311,93 @@ def _city_growth(iso3, anchor_year, population, city_historic, city_future): return _country_growth(iso3, anchor_year, city_historic, city_future) +#: SWEET's own country names, lower-cased, to the codes its defaults are keyed by. +_ISO3_BY_NAME = {name.lower(): iso3 for name, iso3 in defaults_2019.country_to_iso3.items()} + + +def _fold(text): + """``text`` lower-cased and without accents, as pycountry's fuzzy search compares names.""" + return "".join( + ch for ch in unicodedata.normalize("NFKD", text.lower()) if not unicodedata.combining(ch) + ) + + +def _is_named(country, name): + """Whether ``name`` is whole words of one of ``country``'s own names, or their initials. + + Initials count from two letters ("UK", "UAE"), because a one-word name's are + just its first letter. + """ + wanted = _fold(name) + for own in (country._fields.get(field) for field in ("name", "official_name", "common_name")): + if not own: + continue + initials = "".join(ch for ch in own if ch.isupper()) + if len(initials) > 1 and wanted == _fold(initials): + return True + if re.search(rf"(? SplitFractions: + """The city's disposal split with ``existing_gas_pct`` of it going to the + landfills it already has with gas capture: the City DST's "move waste to + existing gas capture" lever. + + The landfill and dumpsite shares without capture take the rest, + ``1 - existing_gas_pct``, in the proportions they had to each other. A city + sending 34% of its waste to capture and 66% to a landfill without it, moved to + 90%, sends the other 10% to that landfill. + + Raises CustomError("INVALID_PARAMETERS") when the share is outside [0, 1], when + the city has no landfill with gas capture to move waste to, and when it has no + site without capture to move waste away to. + """ + if not 0.0 <= existing_gas_pct <= 1.0: + raise CustomError( + "INVALID_PARAMETERS", + "The share of waste to existing gas capture must be between 0 and 1, " + f"not {existing_gas_pct}", + ) + with_capture = split_fractions.landfill_w_capture + without_capture = split_fractions.landfill_wo_capture + split_fractions.dumpsite + if with_capture <= 0: + if existing_gas_pct > 0: + raise CustomError( + "INVALID_PARAMETERS", + "This city has no landfill with gas capture to move waste to", + ) + return split_fractions.model_copy() + if without_capture <= 0: + if existing_gas_pct < 1: + raise CustomError( + "INVALID_PARAMETERS", + "This city sends all of its waste to landfills with gas capture, " + "so there is no site without capture to move waste to", + ) + return split_fractions.model_copy() + scale = (1 - existing_gas_pct) / without_capture + return SplitFractions( + landfill_w_capture=existing_gas_pct, + landfill_wo_capture=split_fractions.landfill_wo_capture * scale, + dumpsite=split_fractions.dumpsite * scale, + new_w_capture=split_fractions.new_w_capture, + ) + + class City: def __init__(self, city_name: str): """ @@ -803,6 +1002,17 @@ def load_csv_new(self, db: pd.DataFrame, scenario: int = 0, dst: bool = False) - other=city_data["Waste Components: Other (%)"].values[0] / 100, ) waste_fractions_dict = waste_fractions.model_dump() + # A published composition need not add up to 100%: Brazil's precomputed + # rows carry their source percentages as they are (São Paulo's sum to 99%, + # Brasília's to 87%, Brazil's national default to 100.1%). Normalize, as + # the other loaders do. Otherwise the typed masses below miss or exceed + # the city's waste, and implement_dst_changes_simple_v1_5 rejects a + # composition more than 1% off. + total = sum(waste_fractions_dict.values()) + if total > 0: # an empty or missing composition stays as published + waste_fractions_dict = { + waste: fraction / total for waste, fraction in waste_fractions_dict.items() + } waste_fractions = pd.DataFrame(waste_fractions_dict, index=years) div_fractions = DiversionFractions( @@ -1426,7 +1636,7 @@ def load_andre_params(self, row, backfill=False): } self.landfill_split_defaults = True else: - if region in defaults_2019.landfill_default_regions: + if site_type_defaults.landfill_is_default(iso3): split_fractions = { "landfill_w_capture": 0, "landfill_wo_capture": 1, @@ -1517,7 +1727,7 @@ def load_andre_params(self, row, backfill=False): } self.landfill_split_defaults = True else: - if region in defaults_2019.landfill_default_regions: + if site_type_defaults.landfill_is_default(iso3): split_fractions = { "landfill_w_capture": 0.0, "landfill_wo_capture": 1.0, @@ -1549,7 +1759,7 @@ def load_andre_params(self, row, backfill=False): # Normalize landfills to 1 split_total = sum([x for x in split_fractions.values()]) if split_total == 0: - if region in defaults_2019.landfill_default_regions: + if site_type_defaults.landfill_is_default(iso3): split_fractions = { "landfill_w_capture": 0.0, "landfill_wo_capture": 1.0, @@ -2464,17 +2674,7 @@ def site_only_estimate(self, row=None, pop_data=None): depth = None site_type = row["Site Type"].values[0] if site_type not in get_site_type_idx.keys(): - if self.region in [ - "Australia and New Zealand", - "Eastern Asia", - "North America", - "Northern Europe", - "Southern Europe", - "Western Europe" - ]: - site_type = "Sanitary Landfill" - else: - site_type = "Dumpsite" + site_type = site_type_defaults.default_site_type(self.iso3) site_type_idx = get_site_type_idx[site_type] city_params_dict = baseline.update_cityparams_dict() baseline.city_params_dict = city_params_dict @@ -2489,6 +2689,7 @@ def site_only_estimate(self, row=None, pop_data=None): gas_capture_efficiency = gas_eff_options[site_type] mcf = mcf_defaults.mcf_for_site(site_type_idx, depth) open_date = row['Site Open Year'].fillna(MODEL_START_YEAR).values[0] + # decision: waste-before-1970-is-zero if open_date < MODEL_START_YEAR: open_date = MODEL_START_YEAR close_date = row['Site Close Year'].fillna(MODEL_END_YEAR + 1).values[0] @@ -2496,6 +2697,7 @@ def site_only_estimate(self, row=None, pop_data=None): 0.0, index=self.years_range ) # Sites accept waste in open years but not the closure year. + # decision: intake-ends-before-close-year fration_of_waste_vector.loc[open_date:close_date-1] = 1.0 new_landfill = Landfill( open_date=open_date, @@ -2667,22 +2869,9 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po depth = canonical_row['waste_depth'] site_type = canonical_row['type'] if pd.isna(site_type) or site_type == '': - if self.region in defaults_2019.landfill_default_regions: - site_type = "Sanitary Landfill" - else: - site_type = "Dumpsite" - if site_type not in get_site_type_idx.keys(): - if self.region in [ - "Australia and New Zealand", - "Eastern Asia", - "North America", - "Northern Europe", - "Southern Europe", - "Western Europe" - ]: - site_type = "Sanitary Landfill" - else: - site_type = "Dumpsite" + site_type = site_type_defaults.default_site_type( + self.iso3, canonical_row.get("income_id") + ) site_type_idx = get_site_type_idx[site_type] city_params_dict = baseline.update_cityparams_dict() baseline.landfills = [] @@ -2709,6 +2898,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po open_date = MODEL_START_YEAR else: open_date = int(open_date) + # decision: waste-before-1970-is-zero if open_date < MODEL_START_YEAR: open_date = MODEL_START_YEAR if open_date == 20007: @@ -2727,6 +2917,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po fraction_of_waste_vector = pd.Series( 0.0, index=self.years_range ) + # decision: intake-ends-before-close-year fraction_of_waste_vector.loc[open_date:close_date-1] = 1.0 id = int(canonical_row['asset_identifier']) oxidation_series = _build_oxidation_series( @@ -2737,6 +2928,7 @@ def site_only_estimate_trace(self, canonical_row=None, time_series_rows=None, po # mitigation raises it to 0.98/0.99 via _gccs_flaring (max/clip). Local import # avoids the dst_common <-> city_params import cycle. from SWEET_python.dst_common import DEFAULT_FLARE_EFFICIENCY + # decision: flaring-uniform flaring_series = pd.Series(DEFAULT_FLARE_EFFICIENCY, index=self.years_range) new_landfill = Landfill( open_date=open_date, @@ -2904,22 +3096,9 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit depth = canonical_row['waste_depth'] site_type = canonical_row['type'] if pd.isna(site_type) or site_type == '': - if self.region in defaults_2019.landfill_default_regions: - site_type = "Sanitary Landfill" - else: - site_type = "Dumpsite" - if site_type not in get_site_type_idx.keys(): - if self.region in [ - "Australia and New Zealand", - "Eastern Asia", - "North America", - "Northern Europe", - "Southern Europe", - "Western Europe" - ]: - site_type = "Sanitary Landfill" - else: - site_type = "Dumpsite" + site_type = site_type_defaults.default_site_type( + self.iso3, canonical_row.get("income_id") + ) site_type_idx = get_site_type_idx[site_type] city_params_dict = baseline.update_cityparams_dict() baseline.landfills = [] @@ -2949,6 +3128,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit open_date = MODEL_START_YEAR else: open_date = int(open_date) + # decision: waste-before-1970-is-zero if open_date < MODEL_START_YEAR: open_date = MODEL_START_YEAR if open_date == 20007: @@ -2973,6 +3153,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit # (see site_only_estimate_trace). Applies to both the single- and multi-city # landfill constructors below. Local import avoids the dst_common import cycle. from SWEET_python.dst_common import DEFAULT_FLARE_EFFICIENCY + # decision: flaring-uniform flaring_series = pd.Series(DEFAULT_FLARE_EFFICIENCY, index=self.years_range) baseline._singapore_k(advanced_baseline=True) if (citysite_rows is None) or (isinstance(citysite_rows, pd.Series)): @@ -2980,6 +3161,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit 0.0, index=self.years_range ) # Sites accept waste in open years but not the closure year. + # decision: intake-ends-before-close-year fraction_of_waste_vector.loc[open_date:close_date-1] = 1.0 new_landfill = Landfill( open_date=open_date, @@ -3079,6 +3261,7 @@ def citysite_estimate_trace(self, canonical_row=None, time_series_rows=None, cit # per-city fractions are the only deposition gate for the virtual # landfills below, so zero them outside the operating window. operating_years = pd.Series(False, index=self.years_range) + # decision: intake-ends-before-close-year operating_years.loc[open_date:close_date-1] = True fraction_of_waste_df.loc[~operating_years] = 0.0 baseline.fraction_of_waste_df = fraction_of_waste_df @@ -3390,6 +3573,7 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s general_reference_year = current_year - 1 # Check if row is a single row or multiple rows + # decision: intake-follows-population if (time_series_rows is None) or isinstance(time_series_rows, pd.Series): # Single row - use old method year_of_data_msw = canonical_row['incoming_waste_year'] @@ -3429,6 +3613,7 @@ def import_basics_site(self, canonical_row, pop_data, usecase='wastemap', time_s waste_per_capita = waste_mass * 1000 / population / 365 else: + # decision: intake-follows-population # Multiple rows - create time series waste_mass_defaults = False year_of_data_msw = time_series_rows['incoming_waste_year'].max() @@ -3692,6 +3877,7 @@ def _is_transient_db_error(err: Exception) -> bool: # Normalize waste fractions to sum to 1 wf_norm = waste_fractions / waste_fractions.sum() + # decision: waste-composition waste_fractions = pd.DataFrame( np.repeat(wf_norm.values[np.newaxis, :], len(self.years_range), axis=0), index=self.years_range, @@ -4725,40 +4911,11 @@ def dst_baseline_blank( textiles=pd.Series(ks.get("textiles", 0.0), index=years), ) - # Determine waste split fractions using .get() method - dumpsite_frac = defaults_2019.fraction_open_dumped_country.get( - iso3, defaults_2019.fraction_open_dumped.get(region, 0) - ) - landfill_wo_capture_frac = defaults_2019.fraction_landfilled_country.get( - iso3, defaults_2019.fraction_landfilled.get(region, 0) + # Determine waste split fractions, as the cities table does for a city + # with no landfill data + split_fractions = SplitFractions( + **defaults_2019.disposal_split_for(iso3, region) ) - landfill_w_capture_frac = 0.0 # Default as per original function - - try: - split_fractions = SplitFractions( - dumpsite=dumpsite_frac, - landfill_wo_capture=landfill_wo_capture_frac, - landfill_w_capture=landfill_w_capture_frac, - ) - except KeyError: - if self.region in defaults_2019.landfill_default_regions: - split_fractions = SplitFractions( - landfill_w_capture=0.0, landfill_wo_capture=1.0, dumpsite=0.0 - ) - else: - split_fractions = SplitFractions( - landfill_w_capture=0.0, landfill_wo_capture=0.0, dumpsite=1.0 - ) - - # Normalize split fractions - split_total = sum(split_fractions.model_dump().values()) - if split_total > 0: - split_fractions = SplitFractions( - **{ - site: frac / split_total - for site, frac in split_fractions.model_dump().items() - } - ) # Instantiate landfill objects years_range = range(MODEL_START_YEAR, MODEL_END_YEAR + 1) @@ -7201,21 +7358,25 @@ def implement_dst_changes_simple_v1_5( mcf_defaults.MCF_UNCATEGORISED, index=range(MODEL_START_YEAR, MODEL_END_YEAR + 1), ) + # Keep these oxidation factors: Landfill.estimate_emissions would + # reset them from gas_capture, which is still False for the two + # converted sites, to 0.1 and 0. skip_ox = True if move_gas: - original_gas_pct = ( - scenario_parameters.split_fractions.landfill_w_capture + scenario_parameters.split_fractions = _move_to_existing_gas_capture( + scenario_parameters.split_fractions, existing_gas_pct + ) + # Each landfill's waste comes from its own fraction_of_waste + # (LandfillWasteMassDF.create below), not from split_fractions. + split = scenario_parameters.split_fractions + scenario_parameters.landfills[0].fraction_of_waste = ( + split.landfill_w_capture ) - scenario_parameters.split_fractions.landfill_w_capture = ( - existing_gas_pct + scenario_parameters.landfills[1].fraction_of_waste = ( + split.landfill_wo_capture ) - if original_gas_pct > 0: - ratio = existing_gas_pct / original_gas_pct - else: - ratio = 0 - scenario_parameters.split_fractions.landfill_wo_capture *= ratio - scenario_parameters.split_fractions.dumpsite *= ratio + scenario_parameters.landfills[2].fraction_of_waste = split.dumpsite if new_gas_pct > 0: scenario_parameters.landfills.append( @@ -7232,7 +7393,8 @@ def implement_dst_changes_simple_v1_5( total = sum(scenario_parameters.split_fractions.model_dump().values()) if abs(total - 1.0) > 1e-3: raise CustomError( - f"Invalid split fractions: {scenario_parameters.split_fractions}" + "INVALID_PARAMETERS", + f"Invalid split fractions: {scenario_parameters.split_fractions}", ) scenario_parameters.landfills[0].fraction_of_waste = ( scenario_parameters.split_fractions.landfill_w_capture @@ -7251,6 +7413,7 @@ def implement_dst_changes_simple_v1_5( lf.scenario = 1 else: + skip_ox = True if add_gas: for lf in scenario_parameters.landfills: if lf.site_type == "Sanitary Landfill": @@ -7538,7 +7701,7 @@ def fill_missing_fields(d: dict) -> dict: ) # scenario_parameters.repopulate_attr_dicts() # does this need to come sooner? Does anything in the above functions rely on the attr dicts? - skip_ox = scenario_parameters.sites_method + # skip_ox comes from the branch above, so the oxidation it set is kept. for landfill in scenario_parameters.landfills: landfill.estimate_emissions(skip_ox=skip_ox) # print(landfill.emissions) @@ -9067,18 +9230,26 @@ async def sdst_prepopulate( raise ValueError(f"Region for ISO3 code '{iso3}' not found.") else: location = geolocator.reverse((latlon[0], latlon[1]), language="en") - country = location.raw["address"].get("country") + # geopy gives None where Nominatim can't place the point (open sea), and + # an address can lack a country (Antarctica). + address = location.raw.get("address", {}) if location is not None else {} + country = address.get("country") try: - iso3 = _iso3_for(country) + iso3 = _iso3_at(address) except LookupError: + if country is None: + raise ValueError(f"No country at {latlon[0]}, {latlon[1]}.") raise ValueError(f"Country '{country}' not found.") region = defaults_2019.region_lookup_iso3.get(iso3) if region is None: raise ValueError(f"Region for ISO3 code '{iso3}' not found.") - if region in defaults_2019.landfill_default_regions: + if site_type_defaults.landfill_is_default(iso3): site_type = "Landfill" else: site_type = "Dumpsite" + # A site in the sites list with no recorded type takes the same rule. + if pd.isna(site_type) or site_type == "": + site_type = site_type_defaults.default_site_type(iso3) if site_type in ["Landfill", "Sanitary Landfill"]: site_type = 0 diff --git a/SWEET_python/class_defs.py b/SWEET_python/class_defs.py index f2f8752..149ca94 100644 --- a/SWEET_python/class_defs.py +++ b/SWEET_python/class_defs.py @@ -31,6 +31,7 @@ def growth_factors_for_years( population Series serves any pivot year, e.g. an ``implement_year`` scenario. """ years = np.asarray(years) + # decision: population-growth if population_series is not None and len(population_series) > 0: P = population_series.reindex(years) # Hold the nearest known value across any gap so the factor never goes NaN. diff --git a/SWEET_python/constants.py b/SWEET_python/constants.py index 84c7f77..e01fa30 100644 --- a/SWEET_python/constants.py +++ b/SWEET_python/constants.py @@ -24,15 +24,15 @@ population-driven, and city_params._population_series_from_pop_data returns None unless the table carries EVERY column from MODEL_START_YEAR onward -- which silently drops every country back to the frozen-CAGR growth scalars. Regenerate -with diagnostic_scripts/generate_pops_yearly.py and upload to blob -static_data/pops_yearly.csv BEFORE the constant lands in a run. - -THERE ARE TWO COPIES OF THAT TABLE, and nothing keeps them in step. Climate TRACE's -pipeline reads the blob copy; the WasteMAP tools read SWEET's own -SWEET_python/pops_yearly.csv (SWEET_python.population), which is the same table -extended back to 1950 because the site tool models from a landfill's real opening -year. Regenerate both from the same WPP file: the generator starts at -MODEL_START_YEAR, so SWEET's copy needs its START_YEAR set to 1950. +SWEET_python/pops_yearly.csv with Climate TRACE's +diagnostic_scripts/generate_pops_yearly.py BEFORE the constant lands in a run. + +THERE IS ONE COPY OF THAT TABLE: SWEET_python/pops_yearly.csv (SWEET_python.population), +from 1950, because the site tool models from a landfill's real opening year. Climate +TRACE's pipeline reads this file too, MODEL_START_YEAR onward (its +util/population.py), and so does its offline country-data build; the blob copy it used +to read, static_data/pops_yearly.csv, is no longer read by anything. So which +population a run used is fixed by the SWEET commit the run records. """ MODEL_START_YEAR: int = 1970 diff --git a/SWEET_python/country_income.py b/SWEET_python/country_income.py new file mode 100644 index 0000000..57f317a --- /dev/null +++ b/SWEET_python/country_income.py @@ -0,0 +1,241 @@ +"""World Bank income group by country, as Climate TRACE reads it. + +Copied from ``raw_emissions.raw_world_bank_country_level`` in the WasteMAP +database on 2026-10-13. That table is loaded, unchanged, from +``country_level_data_0.csv`` in the ``worldbank`` blob container, which is the +country file of the World Bank's What a Waste 2.0 dataset (2018). Its income +groups are the World Bank classification of that time, not the current one: +Argentina is high income here, and Bulgaria and Costa Rica are not. + +Keep this in step with that table rather than with the latest World Bank list. +Climate TRACE's pipeline reads the table for every site, and the point of +:mod:`SWEET_python.site_type_defaults` is that every tool reaches the same +answer for the same country. + +Countries the table does not list (overseas departments, uninhabited +territories, North Korea, the Holy See) are absent here too, so they have no +income group. +""" + +from typing import Dict + +#: ``HIC`` high, ``UMC`` upper-middle, ``LMC`` lower-middle, ``LIC`` low. +INCOME_ID_BY_ISO3: Dict[str, str] = { + "ABW": "HIC", # Aruba + "AFG": "LIC", # Afghanistan + "AGO": "LMC", # Angola + "ALB": "UMC", # Albania + "AND": "HIC", # Andorra + "ARE": "HIC", # United Arab Emirates + "ARG": "HIC", # Argentina + "ARM": "UMC", # Armenia + "ASM": "UMC", # American Samoa + "ATG": "HIC", # Antigua and Barbuda + "AUS": "HIC", # Australia + "AUT": "HIC", # Austria + "AZE": "UMC", # Azerbaijan + "BDI": "LIC", # Burundi + "BEL": "HIC", # Belgium + "BEN": "LIC", # Benin + "BFA": "LIC", # Burkina Faso + "BGD": "LMC", # Bangladesh + "BGR": "UMC", # Bulgaria + "BHR": "HIC", # Bahrain + "BHS": "HIC", # Bahamas, The + "BIH": "UMC", # Bosnia and Herzegovina + "BLR": "UMC", # Belarus + "BLZ": "UMC", # Belize + "BMU": "HIC", # Bermuda + "BOL": "LMC", # Bolivia + "BRA": "UMC", # Brazil + "BRB": "HIC", # Barbados + "BRN": "HIC", # Brunei Darussalam + "BTN": "LMC", # Bhutan + "BWA": "UMC", # Botswana + "CAF": "LIC", # Central African Republic + "CAN": "HIC", # Canada + "CHE": "HIC", # Switzerland + "CHI": "HIC", # Channel Islands + "CHL": "HIC", # Chile + "CHN": "UMC", # China + "CIV": "LMC", # Côte d’Ivoire + "CMR": "LMC", # Cameroon + "COD": "LIC", # Congo, Dem. Rep. + "COG": "LMC", # Congo, Rep. + "COL": "UMC", # Colombia + "COM": "LIC", # Comoros + "CPV": "LMC", # Cabo Verde + "CRI": "UMC", # Costa Rica + "CUB": "UMC", # Cuba + "CUW": "HIC", # Curacao + "CYM": "HIC", # Cayman Islands + "CYP": "HIC", # Cyprus + "CZE": "HIC", # Czech Republic + "DEU": "HIC", # Germany + "DJI": "LMC", # Djibouti + "DMA": "UMC", # Dominica + "DNK": "HIC", # Denmark + "DOM": "UMC", # Dominican Republic + "DZA": "UMC", # Algeria + "ECU": "UMC", # Ecuador + "EGY": "LMC", # Egypt, Arab Rep. + "ERI": "LIC", # Eritrea + "ESP": "HIC", # Spain + "EST": "HIC", # Estonia + "ETH": "LIC", # Ethiopia + "FIN": "HIC", # Finland + "FJI": "UMC", # Fiji + "FRA": "HIC", # France + "FRO": "HIC", # Faeroe Islands + "FSM": "LMC", # Micronesia, Fed. Sts. + "GAB": "UMC", # Gabon + "GBR": "HIC", # United Kingdom + "GEO": "LMC", # Georgia + "GHA": "LMC", # Ghana + "GIB": "HIC", # Gibraltar + "GIN": "LIC", # Guinea + "GMB": "LIC", # Gambia, The + "GNB": "LIC", # Guinea-Bissau + "GNQ": "UMC", # Equatorial Guinea + "GRC": "HIC", # Greece + "GRD": "UMC", # Grenada + "GRL": "HIC", # Greenland + "GTM": "UMC", # Guatemala + "GUM": "HIC", # Guam + "GUY": "UMC", # Guyana + "HKG": "HIC", # Hong Kong SAR, China + "HND": "LMC", # Honduras + "HRV": "HIC", # Croatia + "HTI": "LIC", # Haiti + "HUN": "HIC", # Hungary + "IDN": "LMC", # Indonesia + "IMN": "HIC", # Isle of Man + "IND": "LMC", # India + "IRL": "HIC", # Ireland + "IRN": "UMC", # Iran, Islamic Rep. + "IRQ": "UMC", # Iraq + "ISL": "HIC", # Iceland + "ISR": "HIC", # Israel + "ITA": "HIC", # Italy + "JAM": "UMC", # Jamaica + "JOR": "UMC", # Jordan + "JPN": "HIC", # Japan + "KAZ": "UMC", # Kazakhstan + "KEN": "LMC", # Kenya + "KGZ": "LMC", # Kyrgyz Republic + "KHM": "LMC", # Cambodia + "KIR": "LMC", # Kiribati + "KNA": "HIC", # St. Kitts and Nevis + "KOR": "HIC", # Korea, Rep. + "KWT": "HIC", # Kuwait + "LAO": "LMC", # Lao PDR + "LBN": "UMC", # Lebanon + "LBR": "LIC", # Liberia + "LBY": "UMC", # Libya + "LCA": "UMC", # St. Lucia + "LIE": "HIC", # Liechtenstein + "LKA": "LMC", # Sri Lanka + "LSO": "LMC", # Lesotho + "LTU": "HIC", # Lithuania + "LUX": "HIC", # Luxembourg + "LVA": "HIC", # Latvia + "MAC": "HIC", # Macao SAR, China + "MAF": "HIC", # St. Martin (French part) + "MAR": "LMC", # Morocco + "MCO": "HIC", # Monaco + "MDA": "LMC", # Moldova + "MDG": "LIC", # Madagascar + "MDV": "UMC", # Maldives + "MEX": "UMC", # Mexico + "MHL": "UMC", # Marshall Islands + "MKD": "UMC", # Macedonia, FYR + "MLI": "LIC", # Mali + "MLT": "HIC", # Malta + "MMR": "LMC", # Myanmar + "MNE": "UMC", # Montenegro + "MNG": "LMC", # Mongolia + "MNP": "HIC", # Northern Mariana Islands + "MOZ": "LIC", # Mozambique + "MRT": "LMC", # Mauritania + "MUS": "UMC", # Mauritius + "MWI": "LIC", # Malawi + "MYS": "UMC", # Malaysia + "NAM": "UMC", # Namibia + "NCL": "HIC", # New Caledonia + "NER": "LIC", # Niger + "NGA": "LMC", # Nigeria + "NIC": "LMC", # Nicaragua + "NLD": "HIC", # Netherlands + "NOR": "HIC", # Norway + "NPL": "LIC", # Nepal + "NRU": "UMC", # Nauru + "NZL": "HIC", # New Zealand + "OMN": "HIC", # Oman + "PAK": "LMC", # Pakistan + "PAN": "HIC", # Panama + "PER": "UMC", # Peru + "PHL": "LMC", # Philippines + "PLW": "HIC", # Palau + "PNG": "LMC", # Papua New Guinea + "POL": "HIC", # Poland + "PRI": "HIC", # Puerto Rico + "PRT": "HIC", # Portugal + "PRY": "UMC", # Paraguay + "PSE": "LMC", # West Bank and Gaza + "PYF": "HIC", # French Polynesia + "QAT": "HIC", # Qatar + "ROU": "UMC", # Romania + "RUS": "UMC", # Russian Federation + "RWA": "LIC", # Rwanda + "SAU": "HIC", # Saudi Arabia + "SDN": "LMC", # Sudan + "SEN": "LIC", # Senegal + "SGP": "HIC", # Singapore + "SLB": "LMC", # Solomon Islands + "SLE": "LIC", # Sierra Leone + "SLV": "LMC", # El Salvador + "SMR": "HIC", # San Marino + "SOM": "LIC", # Somalia + "SRB": "UMC", # Serbia + "SSD": "LIC", # South Sudan + "STP": "LMC", # São Tomé and Príncipe + "SUR": "UMC", # Suriname + "SVK": "HIC", # Slovak Republic + "SVN": "HIC", # Slovenia + "SWE": "HIC", # Sweden + "SWZ": "LMC", # Eswatini + "SXM": "HIC", # Sint Maarten (Dutch part) + "SYC": "HIC", # Seychelles + "SYR": "LIC", # Syrian Arab Republic + "TCA": "HIC", # Turks and Caicos Islands + "TCD": "LIC", # Chad + "TGO": "LIC", # Togo + "THA": "UMC", # Thailand + "TJK": "LIC", # Tajikistan + "TKM": "UMC", # Turkmenistan + "TLS": "LMC", # Timor-Leste + "TON": "UMC", # Tonga + "TTO": "HIC", # Trinidad and Tobago + "TUN": "LMC", # Tunisia + "TUR": "UMC", # Turkey + "TUV": "UMC", # Tuvalu + "TWN": "HIC", # NA + "TZA": "LIC", # Tanzania + "UGA": "LIC", # Uganda + "UKR": "LMC", # Ukraine + "URY": "HIC", # Uruguay + "USA": "HIC", # United States + "UZB": "LMC", # Uzbekistan + "VCT": "UMC", # St. Vincent and the Grenadines + "VEN": "UMC", # Venezuela, RB + "VGB": "HIC", # British Virgin Islands + "VIR": "HIC", # Virgin Islands (U.S.) + "VNM": "LMC", # Vietnam + "VUT": "LMC", # Vanuatu + "WSM": "UMC", # Samoa + "XKX": "LMC", # Kosovo + "YEM": "LIC", # Yemen, Rep. + "ZAF": "UMC", # South Africa + "ZMB": "LMC", # Zambia + "ZWE": "LIC", # Zimbabwe +} diff --git a/SWEET_python/defaults_2019.py b/SWEET_python/defaults_2019.py index 83ff96e..325a9c1 100644 --- a/SWEET_python/defaults_2019.py +++ b/SWEET_python/defaults_2019.py @@ -412,6 +412,7 @@ def waste_composition_for(iso3, region): plastic = candidate.get("plastic") if pd.notna(plastic) and plastic > 0: row = candidate + # decision: waste-composition if row is None: row = waste_fraction_defaults.loc[region, :] total = row.sum() @@ -686,6 +687,7 @@ def waste_composition_for(iso3, region): for country in region_lookup: iso3 = country_to_iso3[country] region_lookup_iso3[iso3] = region_lookup[country] +# tonnes/cap/yr: IPCC 2019 Refinement, Vol. 5 Ch. 2, Table 2.1 (Updated) msw_per_capita_defaults = { "Australia and New Zealand": 0.6, "Caribbean": 0.78, @@ -712,6 +714,7 @@ def waste_composition_for(iso3, region): msw_per_capita_defaults = { key: (value / 365) * 1000 for key, value in msw_per_capita_defaults.items() } # convert from tons/year to kg/day +# tonnes/cap/yr: IPCC 2019 Refinement, Vol. 5 Ch. 2, Table 2A.1 (Updated), 2010 column msw_per_capita_country = { "Tajikistan": 0.32, "Turkmenistan": 0.36, @@ -720,6 +723,7 @@ def waste_composition_for(iso3, region): "Mongolia": 0.24, "South Korea": 0.35, "Brunei Darussalam": 0.32, + "Indonesia": 0.19, "Laos": 0.26, "Malaysia": 0.55, "Myanmar": 0.16, @@ -1284,6 +1288,53 @@ def get_precipitation_zone(rainfall): country_to_iso3[country]: value for country, value in fraction_landfilled_country.items() } + + +def disposal_split_for(iso3, region): + """Default disposal split for a city with no landfill data of its own. + + Returns the shares of disposed waste sent to ``landfill_w_capture``, + ``landfill_wo_capture`` and ``dumpsite``, summing to 1. The country's row is + used when it sends waste to a landfill or a dump, else the region's. A place + with neither is all landfill where a site of unknown type is assumed a + sanitary landfill (``site_type_defaults.landfill_is_default``, Climate TRACE's + rule: high income, or a region or country it names), and all dumpsite + elsewhere. + + Southern Asia, South-Eastern Asia and Southern Africa have no regional row + (the other regional tables carry them as ``0.0 # np.nan``), so their + countries without a row of their own take that fallback. Canada's, Germany's + and Switzerland's rows are all zeros, so they take their region's, which is + landfill only. + + This is the split ``City.load_andre_params`` gives a cities-table row with no + landfill data, so a Custom Location starts from the same split as a mapped + city in the same country. + """ + for key, dumped, landfilled in ( + (iso3, fraction_open_dumped_country, fraction_landfilled_country), + (region, fraction_open_dumped, fraction_landfilled), + ): + dumpsite = dumped.get(key, 0.0) + landfill = landfilled.get(key, 0.0) + total = dumpsite + landfill + if total > 0: + return { + "landfill_w_capture": 0.0, + "landfill_wo_capture": landfill / total, + "dumpsite": dumpsite / total, + } + # Imported here: site_type_defaults imports this module. + from SWEET_python.site_type_defaults import landfill_is_default + + landfills = landfill_is_default(iso3) + return { + "landfill_w_capture": 0.0, + "landfill_wo_capture": 1.0 if landfills else 0.0, + "dumpsite": 0.0 if landfills else 1.0, + } + + fraction_incinerated_country = { "Japan": 0.76, "South Korea": 0.22, diff --git a/SWEET_python/mcf.py b/SWEET_python/mcf.py index 9c81e99..fea8437 100644 --- a/SWEET_python/mcf.py +++ b/SWEET_python/mcf.py @@ -108,6 +108,7 @@ def mcf_for_site(site_type_idx: int, depth: Optional[float] = None) -> float: Returns: The methane correction factor, between 0 and 1. """ + # decision: mcf-by-type-and-depth if site_type_idx not in DEPTH_SENSITIVE_TYPES: return MCF_BY_TYPE[site_type_idx] if depth is None or pd.isna(depth): diff --git a/SWEET_python/model_v2.py b/SWEET_python/model_v2.py index 4bccf29..1448a5e 100644 --- a/SWEET_python/model_v2.py +++ b/SWEET_python/model_v2.py @@ -123,6 +123,7 @@ def estimate_emissions2(self): mcf_values = mcf.loc[year_range].values[:, None] ch4_produce = ( ks_values + # decision: methane-potential-by-waste-type * defaults_2019.L_0[waste] * waste_masses * exp_term @@ -241,6 +242,7 @@ def estimate_emissions_monthly(self): # Month length in years (variable by month and leap years) month_days = monthly_dates.days_in_month.values year_days = np.array([366 if calendar.isleap(y) else 365 for y in years]) + # decision: monthly-from-decay-model month_fracs = month_days / year_days # Expand annual factors to monthly arrays @@ -268,6 +270,7 @@ def expand_to_months(annual_series_or_value): for i, waste in enumerate(components): # Get constants + # decision: methane-potential-by-waste-type L0 = defaults_2019.L_0[waste] # Monthly series @@ -348,6 +351,7 @@ def expand_to_months(annual_series_or_value): # because flare_values represents destruction efficiency. try: flared_destroyed_m3 = captured_df.sum(axis=1).fillna(0.0) + # decision: particulate-factors self.pm2p5_kg_monthly = (flared_destroyed_m3 * PM2P5_KG_PER_M3_CH4_FLARED).astype(float) self.pm10_kg_monthly = (flared_destroyed_m3 * PM10_KG_PER_M3_CH4_FLARED).astype(float) except Exception: diff --git a/SWEET_python/population.py b/SWEET_python/population.py index 295e8df..24125e4 100644 --- a/SWEET_python/population.py +++ b/SWEET_python/population.py @@ -2,12 +2,13 @@ SWEET grows a place's waste with its country's population, year by year: the factor for a year is P(year) / P(anchor year) (see class_defs.growth_factors_for_years). -Climate TRACE's pipeline hands SWEET the same numbers through ``pop_data``; the -WasteMAP tools, which have no ``pop_data``, read them here. +The WasteMAP tools, which have no ``pop_data``, read them here. Climate TRACE's +pipeline reads this same file (1970 onward, in its ``util/population.py``) and hands +SWEET those numbers through ``pop_data``, so there is one copy of the table, versioned +with this package. -``pops_yearly.csv`` is Climate TRACE's ``static_data/pops_yearly.csv`` (1970-2050) -unchanged, extended back to 1950 from the same source, because the site tool models -a landfill from its real opening year. Built with Climate TRACE's +``pops_yearly.csv`` runs from 1950 because the site tool models a landfill from its +real opening year. Built with Climate TRACE's ``diagnostic_scripts/generate_pops_yearly.py`` from ``WPP2024_GEN_F01_DEMOGRAPHIC_INDICATORS_COMPACT.xlsx``, "Total Population, as of 1 January (thousands)": estimates to 2023, medium variant after. diff --git a/SWEET_python/singapore_k/singapore_k.py b/SWEET_python/singapore_k/singapore_k.py index 3cec129..52424ad 100644 --- a/SWEET_python/singapore_k/singapore_k.py +++ b/SWEET_python/singapore_k/singapore_k.py @@ -380,6 +380,7 @@ def _create_series(kc, tf, fm, implement_year=None, advanced_baseline=False, # Floor the decomposition rate at the IPCC dry-boreal low-end default. # Cold/dry sites can otherwise drive k toward 0 (tf -> 0 near ambient # -10 C); clip leaves any genuine NaN entries untouched. See K_MIN_PER_YEAR. + # decision: decay-rate-from-climate years = years.clip(lower=K_MIN_PER_YEAR) return years diff --git a/SWEET_python/site_type_defaults.py b/SWEET_python/site_type_defaults.py new file mode 100644 index 0000000..fa6255c --- /dev/null +++ b/SWEET_python/site_type_defaults.py @@ -0,0 +1,102 @@ +"""The site type to assume for a site whose type nobody recorded. + +Every SWEET path that has to guess a site type calls :func:`default_site_type` +or :func:`landfill_is_default`, and so do Climate TRACE's pipeline and WasteMAP. +Before 2026-10-13 the guess was made in nine places by three different rules, +so a Kazakh site with no type was a sanitary landfill in Climate TRACE and an +open dump in WasteMAP's site tool. Do not reintroduce a local copy. + +The rule is Climate TRACE's, moved here unchanged from +``landfill_table_ops.fill_landfill_types``. A site is assumed to be a sanitary +landfill when its country's disposal system is engineered, and an open dump +otherwise. That is a claim about the national system, which is knowable even +when the individual site is in no registry: there are no unmanaged waste piles +in Denmark or the United States. A country counts as engineered if any of these +holds: + +- **It is high income** in the World Bank grouping Climate TRACE reads + (:mod:`SWEET_python.country_income`). +- **Its SWEET region is in** :data:`LANDFILL_DEFAULT_REGIONS`. That is + ``defaults_2019.landfill_default_regions`` plus Western Asia and Central Asia. + Central Asia joins on the argument that admits Russia: the post-Soviet + republics inherited one engineered "polygon" standard, so splitting + Kazakhstan from Russia would be arbitrary. +- **It is named in** :data:`LANDFILL_DEFAULT_COUNTRIES`. Mexico, Brazil and + Costa Rica are there because their own disposal statistics say engineered + landfill while their region does not (What a Waste 3.0: Mexico open-dumps 3% + of its waste, Brazil 13%). China and Russia already qualify by region; naming + them keeps a change to SWEET's region map from re-typing the two largest + inventories. + +:data:`DUMPSITE_COUNTRIES` overrides all three. Mongolia (Central Asia in +SWEET's map) and Yemen (Western Asia) are swept in by a region whose average +does not describe them. + +A country SWEET cannot place in a region, or no country at all, is an open dump. + +This decides MCF and oxidation, so it is worth about 1.5x on a site's modelled +methane: a sanitary landfill is MCF 1.0 with oxidation 0.10, and an open dump of +unknown depth is MCF 0.6 with no oxidation. +""" + +from typing import Optional + +import pandas as pd + +from SWEET_python import defaults_2019 +from SWEET_python.country_income import INCOME_ID_BY_ISO3 + +SANITARY_LANDFILL = "Sanitary Landfill" +DUMPSITE = "Dumpsite" + +LANDFILL_DEFAULT_REGIONS = frozenset( + set(defaults_2019.landfill_default_regions) | {"Western Asia", "Central Asia"} +) +LANDFILL_DEFAULT_COUNTRIES = frozenset({"CHN", "RUS", "MEX", "BRA", "CRI"}) +DUMPSITE_COUNTRIES = frozenset({"MNG", "YEM"}) + + +def _blank(value) -> bool: + return value is None or (not isinstance(value, str) and pd.isna(value)) or value == "" + + +def income_for(iso3: Optional[str]) -> Optional[str]: + """The World Bank income group SWEET holds for a country, or None.""" + if _blank(iso3): + return None + return INCOME_ID_BY_ISO3.get(str(iso3).upper()) + + +# decision: site-type-fill +def landfill_is_default(iso3: Optional[str], income_id: Optional[str] = None) -> bool: + """True where a site of unknown type should be assumed a sanitary landfill. + + Args: + iso3: The country's ISO 3166 alpha-3 code. + income_id: The caller's own income group for the country, if it has + one. A missing value (None, NaN or empty) falls back to + :func:`income_for`, which reads the same World Bank table. + + Returns: + False for an unknown or unplaceable country. + """ + if _blank(iso3): + return False + iso3 = str(iso3).upper() + if iso3 in DUMPSITE_COUNTRIES: + return False + if _blank(income_id): + income_id = income_for(iso3) + return ( + income_id == "HIC" + or defaults_2019.region_lookup_iso3.get(iso3) in LANDFILL_DEFAULT_REGIONS + or iso3 in LANDFILL_DEFAULT_COUNTRIES + ) + + +def default_site_type(iso3: Optional[str], income_id: Optional[str] = None) -> str: + """``"Sanitary Landfill"`` or ``"Dumpsite"`` for a site whose type is unknown. + + See :func:`landfill_is_default` for the arguments. + """ + return SANITARY_LANDFILL if landfill_is_default(iso3, income_id) else DUMPSITE diff --git a/changelog/2026-10.md b/changelog/2026-10.md index ad52192..5f96c06 100644 --- a/changelog/2026-10.md +++ b/changelog/2026-10.md @@ -1,6 +1,48 @@ # SWEET_python Changelog — October 2026 -**Highlights:** A site in the city-level advanced DST can state its own decay +**Highlights:** Indonesia sizes its waste from its own row of the IPCC table +SWEET copies its per-capita waste rates from, instead of the South-Eastern Asia +default, which was 2.4 times higher (model-output change). + +A site whose type nobody recorded is now guessed by Climate TRACE's rule +(income, region and named countries) in every SWEET path, instead of by one of +two region lists. 55 countries' unknown sites move from open dump to sanitary +landfill in WasteMAP's site tool (model-output change). + +`load_csv_new` normalizes a city's composition, as the other +loaders do. WasteMAP's city DST loads a city from its published cities table +this way, and Brazil's precomputed rows there do not add up to 100% (São Paulo +99%, Brasília 87%). Every city DST scenario for those two cities was rejected as +"Invalid waste fractions" (model-output change). + +A Custom Location city modelled no landfill methane at all in 31 +countries, among them Pakistan, India, Bangladesh, Indonesia, the Philippines, +Vietnam, South Africa, Canada and Germany. Its disposal split came out +(0, 0, 0), so its landfills took none of its waste. It now gets the split the +cities table gives a city with no landfill data (model-output change). + +The City DST's "move waste to existing gas capture" lever now +changes a city's emissions. For a city on the generic landfill split, it worked +out a new split and never gave it to the landfills, so every setting returned the +no-change scenario. Moved to 90% gas capture, Lahore now emits 45% less in 2040 +(model-output change). + +The City DST's "add gas capture" lever now applies the oxidation +it sets, not just the gas capture. Its results fall another 10–13% after the +implementation year: a city that landfills everything ends up 65.3% below the +no-change scenario in 2040 instead of 60% (model-output change). + +A Custom Location in Kosovo builds. `City.dst_baseline_blank` +raised "Country 'XKX' not found." for Kosovo's code, so WasteMAP's city tool +answered 500, and the site tool modelled a point in Kosovo as Serbia. SWEET now +looks a country up in its own table before asking pycountry, which has no Kosovo +(model-output change for a country given by name). + +A blank or junk country is not found rather than modelled as the UK or Italy, +and the site tool's prefill finds the country for a point in Turkey, both Congos +and eight more places it used to fail on. + +A site in the city-level advanced DST can state its own decay rate and oxidation rate, and a new call reports the rates the model would use at each site that states none. No request that leaves them out changes. @@ -37,3 +79,371 @@ at each site that states none. No request that leaves them out changes. overrides, the defaults reproduce the plain run bit for bit, and a test holds them to that. ([#88](https://github.com/RMI/SWEET_python/pull/88), [RMI/WasteMAP#883](https://github.com/RMI/WasteMAP/pull/883)) + +- **Seven more Climate TRACE modelling decisions are marked where SWEET applies them**: + waste before 1970 counted as zero, a site's last intake in the year before it closes, + intake following population, the decay rate from climate, each waste type's methane + potential, the monthly decay model and the flaring particulate factor. Comments only; + no number moves. + +- **The places SWEET applies a Climate TRACE modelling decision carry a comment + naming it**, such as `# decision: mcf-by-type-and-depth`. The list of + decisions is `docs/MODEL_DECISIONS.md` in RMI_Climate_TRACE_Waste_Methane, and + its test checks these comments once a SWEET release with them is pinned there. + Seven decisions are tagged: MCF, site type, waste composition, gas capture, + oxidation, flaring and population growth. Comments only; no number moves. + The README's "Decision comments" section explains them. A new pull-request + template asks whether a change touches a tagged line. + +## Changed + +- **A site with no recorded type is guessed the same way everywhere** + (model-output change). The new + `site_type_defaults.default_site_type(iso3, income_id=None)` holds the rule + Climate TRACE's pipeline already used, moved unchanged from its + `landfill_table_ops.fill_landfill_types`. A site is a sanitary landfill where + its country is high income, or its region is in `landfill_default_regions`, + Western Asia or Central Asia, or it is China, Russia, Mexico, Brazil or Costa + Rica. Otherwise it is an open dump, and Mongolia and Yemen always are. + - Income comes from the new `country_income.INCOME_ID_BY_ISO3`: the World Bank + table Climate TRACE reads (What a Waste 2.0's country file, 217 countries), + not the current World Bank list. + - Before, SWEET guessed from the region alone, using one of two lists. One had + Eastern Europe; the copy in `site_only_estimate` had Australia and New + Zealand instead. + - The site tool's prefill by location and both trace paths: 55 countries + change from open dump to sanitary landfill, among them Turkey, Saudi Arabia, + Kazakhstan, Mexico, Brazil, Argentina, Chile, Australia and the high-income + Caribbean islands. None changes the other way. A site there models at MCF + 1.0 with oxidation 0.10 instead of 0.6 with none, about 1.5 times the methane. + - A listed site with no type in the site tool's prefill follows the rule too; + it was always an open dump. + - `site_only_estimate`: 66 countries change, the 55 plus Eastern Europe. + - The city table's last fallback (no landfill data, and no default split for + the country or the region): only Brunei changes. + - Climate TRACE's pipeline does not move. It already used this rule. + - `dst_baseline_blank` is untouched: #74 rewrites its fallback. + ([#83](https://github.com/RMI/SWEET_python/pull/83)) + +- **Gas collection efficiency and oxidation follow a site's reports year by year + when its rows carry their year.** On rows with a `row_year` column -- which + Climate TRACE's pipeline adds on its consolidated facility input, one row per + site per year -- `_resolve_gas_capture` and `_build_oxidation_series` read each + value as belonging to its row's year. The new `_carried_series` makes one value + the value in every year, and carries several back and forward: the first back to + the start, each forward to the next report, the latest to the end. + - A modelled site, which has no reported-emissions year, now keeps its measured + gas collection efficiency instead of the site-type default. + - Rows without `row_year` keep the old rule: the site's mean, overwritten in + reported years. + - No number moves on today's data: every Climate TRACE site with a measured + efficiency also reports methane, and the consolidated table gives one value + per site. + ([#84](https://github.com/RMI/SWEET_python/pull/84)) + +## Fixed + +- **Indonesia has a per-capita waste rate of its own** (model-output change). + `defaults_2019.msw_per_capita_country` copies IPCC 2019 Refinement, Vol. 5 + Ch. 2, Table 2A.1 (Updated), 2010 column, in tonnes per person per year. All + 167 of its rows match the IPCC's, but Indonesia's, 0.19, was never typed in. + Indonesia fell back to the South-Eastern Asia default, 0.46. The IPCC's source + for the row, What a Waste 2012, agrees: 0.52 kg per person per day is 0.19 t + a year. + - A Custom Location (`dst_baseline_blank`) in Indonesia is sized at 190,000 t + a year per million people, down from 460,000 (−58.7%). Its emissions fall by + the same share once a Custom Location in South-Eastern Asia disposes of its + waste ([#74](https://github.com/RMI/SWEET_python/pull/74)); until then + they are zero either way. + - WasteMAP's June 2026 cities table does not move. All 11 Indonesian cities + have waste figures of their own, so `load_andre_params` never reaches the + default for them. + - Climate TRACE's pipeline does not move. It drops a site with no tonnage + before SWEET runs. + + The comments on `msw_per_capita_defaults` and `msw_per_capita_country` now + name the IPCC tables they come from. + ([#76](https://github.com/RMI/SWEET_python/pull/76)) + +- **`load_csv_new` normalizes the composition it reads** (model-output change). + It was the one loader that took a city's ten `Waste Components` as published. + `load_andre_params`, `dst_baseline_blank` and the site loaders all scale a + composition to sum to 1. WasteMAP's cities table publishes Brazil's + precomputed rows with their source composition as it is: + - São Paulo: 99% + - Brasília: 87% + - Rio de Janeiro: 99.59% + - The national default the other Brazilian rows carry: 100.1% + + Two things followed: + - `implement_dst_changes_simple_v1_5` raised "Invalid waste fractions" for São + Paulo and Brasília, whatever the levers. Its check rejects a composition more + than 1% off, and São Paulo's fails by float rounding alone + (1 − 0.99 = 0.010000000000000009). + - The typed masses `load_csv_new` builds missed or exceeded the city's waste by + the same percentage. The no-change scenario then drifted from the baseline + after the implementation year, for 2,233 cities by up to 5.5e-4. + + Each type now keeps its published share of the reported total. Across all + 5,689 city names of WasteMAP's June 2026 table, before and after: + - São Paulo's and Brasília's scenarios run. No other city's result changes + status. + - The city DST baseline and scenarios scale by 100 ÷ the published sum: + - Brasília +14.9% + - São Paulo +1.0% + - Rio de Janeiro +0.4% + - The 4,691 cities on Brazil's default −0.1% + - The 995 cities whose composition already adds up do not move: 949 are + bit-identical, and the rest are within 1e-15. + - The no-change scenario reproduces the baseline for every city. + + Climate TRACE's pipeline does not call `load_csv_new`, so its outputs do not + move. + ([#73](https://github.com/RMI/SWEET_python/pull/73), + [RMI/WasteMAP#852](https://github.com/RMI/WasteMAP/pull/852)) + +- **A Custom Location buries or dumps its waste in every country** + (model-output change). `City.dst_baseline_blank`, which WasteMAP runs for any + city not in the cities table, looked a country's disposal split up as + `fraction_open_dumped_country.get(iso3, fraction_open_dumped.get(region, 0))`, + and the same for `fraction_landfilled`. Two gaps in those tables made the split + (0, 0, 0): + - **Southern Asia, South-Eastern Asia and Southern Africa have no regional + row.** This is not a label mismatch: every other regional table carries these + three regions under the same names, as `0.0 # np.nan`. None of their 28 + countries here has a row of its own. + - **Canada's, Germany's and Switzerland's country rows are all zeros.** + + The normalization step skipped a zero total, so every landfill's + `fraction_of_waste` was 0 and the waste the city didn't divert left the model. + The `except KeyError` fallback written for this never ran, because `.get` + never raises. + + The new `defaults_2019.disposal_split_for(iso3, region)` uses the country's row + when it sends waste to a landfill or a dump, else the region's. A place with + neither is all landfill where a site of unknown type is assumed a sanitary + landfill (`site_type_defaults.landfill_is_default`, Climate TRACE's rule), and + all dumpsite elsewhere. That is the split `City.load_andre_params` gives a + cities-table row with no landfill data, which a test checks for one country + per region and rule. `load_andre_params` itself is unchanged, so the cities + table doesn't move. + + 2040 emissions for a Custom Location of 2 million people (500 mm, 25 °C): + - **27 of the 28 countries in the three regions dump all their waste.** + Pakistan goes from 0 to 21,156 t CH4, the Philippines to 10,513, South Africa + to 33,478 and India to 5,284. + - **Brunei, which is high income, landfills it**, from 0 to 24,983 t. + - **Canada, Germany and Switzerland landfill it.** Canada goes from 40 to + 74,846 t, Germany from 26 to 29,521 and Switzerland from 36 to 17,263. The + old figures were compost alone. + - **The other 218 countries are bit-identical.** This was checked across all + 249 countries a Custom Location could be built for. Kosovo's `XKX` couldn't + be looked up then; #77 fixes that. + + Tests: `tests/test_custom_location_disposal_split.py`. + ([#74](https://github.com/RMI/SWEET_python/pull/74), + [RMI/WasteMAP#853](https://github.com/RMI/WasteMAP/pull/853)) + +- **Moving waste to a city's existing gas capture changes its emissions** + (model-output change). `City.implement_dst_changes_simple_v1_5` with + `move_gas=1` and `existing_gas_pct` is the City DST's "% of waste diverted to + existing landfill with gas capture" slider. Its generic branch rewrote + `split_fractions`, but each landfill's waste comes from its own + `fraction_of_waste`, and only the "new landfill with gas capture" block copied + the split there. WasteMAP offers this slider only to a city that already has gas + capture, and switches the new-landfill lever off for exactly those cities, so the + two never ran together. On the June 2026 cities table, all 7 cities with a live + slider (Mexico City, Lahore, Santiago, Amman, Tegucigalpa, Cebu City, Tunis) + returned the no-change scenario exactly, at every setting. + + The arithmetic was wrong as well. It set the capture share to `existing_gas_pct` + and multiplied the shares without capture by `existing / original`, scaling them + up rather than down: Lahore's landfill without capture came out at 171% of the + city's waste, and Mexico City's at 3,848%. A city with no capture had both shares + zeroed. With a new landfill with gas capture as well, the inflated split failed + its sum check, and that check built `CustomError` with one argument, so the API + returned a 500. + + The new `_move_to_existing_gas_capture` gives the capture share + `existing_gas_pct`. The landfill and dumpsite without capture share the rest in + the proportions they had, and the scenario applies the split to the landfills + from the implementation year. In 2040: + - Moved to 90% capture: Lahore −45%, Mexico City −43%, Cebu City −30%. + - Moved to 100%: Lahore −53%, Mexico City −48%. + - Moving waste off capture raises emissions. Santiago (98.8% to capture) moved + to 0% emits 96% more. + + Three requests the lever can't carry out now return `INVALID_PARAMETERS` + instead of the no-change scenario: + - a share outside [0, 1]; + - a share above 0 for a city with no landfill with gas capture (WasteMAP doesn't + offer the lever there); + - less than 100% for a city that sends all its waste to capture. WasteMAP + disables the slider for the 41 such cities in the June table and sends 100%, + which leaves the split as it is. + + Nothing else moves. The no-change scenario, the other levers and the 41 cities at + 100% give the same numbers as before. The sum check now raises a proper + `INVALID_PARAMETERS` too. + ([#75](https://github.com/RMI/SWEET_python/pull/75), [RMI/WasteMAP#854](https://github.com/RMI/WasteMAP/pull/854)) + +- **"Add gas capture" applies the oxidation it sets** (model-output change). In + the generic branch of `City.implement_dst_changes_simple_v1_5`, `add_gas` + converts a city's two sites without gas capture after the implementation year: + - the landfill gets 60% gas capture and an oxidation factor of 0.22 (was 0.1) + - the dumpsite becomes a controlled dumpsite with 30% capture and 0.1 (was 0) + + It set `skip_ox = True` so that `Landfill.estimate_emissions` would keep those + factors. The emissions loop at the end of the method then replaced it with + `skip_ox = scenario_parameters.sites_method`, which is false on this branch. + So `estimate_emissions` reset each factor from the site's `gas_capture` flag, + which `add_gas` doesn't set: 0.1 for the landfill and 0 for the dumpsite, in + every year. The gas capture applied; the oxidation never did. Both lines date + from the repository's first commit. + + 0.22 and 0.1 are the values `estimate_emissions` itself gives a landfill and a + controlled dumpsite with gas collection (its `ox_cap` table, and + `defaults_2019.oxidation_factor["with_lfg"]`). The real-sites branch also gives + a sanitary landfill 0.22 when it adds capture. Each branch now sets `skip_ox` + itself, and the loop uses it. The real-sites branch keeps `skip_ox = True`. + + Measured across WasteMAP's June 2026 cities table (5,689 city names) and a + Custom Location in each of 249 countries: gas capture added in 2026, with each + city's own diversions. + - **The 168 cities the UI offers the lever to** (no gas capture, some landfill) + fall another 10.0–13.3% in 2040. Against the no-change scenario: + - Lima and Riyadh, which landfill everything: −60.0% → −65.3% + - Cairo, which dumps 82% of its waste: −37.3% → −43.9% + - **A city that only dumps its waste** goes from −30.0% to −37.0%. That covers + 187 cities, among them Lagos, Karachi, Dhaka and Delhi. The UI doesn't offer + them the lever, but the API runs it. + - **Custom Locations** move the same way in the 218 countries where they + landfill or dump. The other 31 have the empty disposal split that + [#74](https://github.com/RMI/SWEET_python/pull/74) fixes. + - **Brazil's 4,694 precomputed cities don't move yet.** WasteMAP's `load_city` + sends them down the real-sites branch, where `add_gas` changes nothing for + generic sites. Once [RMI/WasteMAP#851](https://github.com/RMI/WasteMAP/pull/851) + sends them down the generic branch, they fall another 10.0–13.3% as well. + - **Nothing else moves.** The no-change scenario is bit-identical for every + city and Custom Location, and so is every `add_gas` result up to and + including the implementation year. Without `add_gas`, `skip_ox` is what it + was, so no other lever changes. No scenario that ran before fails now, or the + reverse. + + Climate TRACE's pipeline doesn't call this method, so its outputs do not move. + + Tests: `tests/test_add_gas_capture_oxidation.py`. + ([#78](https://github.com/RMI/SWEET_python/pull/78), + [RMI/WasteMAP#856](https://github.com/RMI/WasteMAP/pull/856)) + +- **A Custom Location or site in Kosovo is modelled as Kosovo** (model-output + change for a country given by name). `_iso3_for`, which `dst_baseline_blank`, + `cityparams_obj_for_blank_site` and `sdst_prepopulate` use to turn the caller's + country into a code, asked pycountry first. pycountry has no Kosovo: + - **By code it raised.** XKX is the code SWEET's defaults and WasteMAP's + country list key Kosovo by. pycountry raised `LookupError` for it, so + `dst_baseline_blank` and `cityparams_obj_for_blank_site` raised + `ValueError("Country 'XKX' not found.")`. WasteMAP's city tool answered 500 + for a Custom Location in Kosovo, and its site tool did the same for a + Climate TRACE site there. + - **By name it was Serbia.** The fuzzy search resolved "Kosovo" to Serbia, + whose province Kosovo-Metohija matches. `sdst_prepopulate` names the country + with the geocoder's English name, so the site tool's prefill for a point in + Kosovo returned Serbia's code and defaults, and the run that followed + modelled Serbia. The same search resolved "Curacao" to the Netherlands, and + found nothing for Macau, Pitcairn Islands, Svalbard and Jan Mayen Islands, + US Minor Islands, or French Southern and Antarctic Lands. + + `_iso3_for` now tries SWEET's own table first, by code + (`defaults_2019.region_lookup_iso3`) and then by name + (`defaults_2019.country_to_iso3`), ignoring case as pycountry does. Anything + else goes to pycountry as before. Kosovo has no country rows in SWEET's + defaults, so it takes Southern Europe's, and it grows with its own UN + population series. + + Effect: + - **A Custom Location in Kosovo** of 2 million people (500 mm, 25 °C) emits + 38,768 t CH4 in 2040. By code it raised before; by name it was Serbia's + 26,818 t. + - **The other 249 countries WasteMAP offers are bit-identical.** Each code + already resolved to itself. + - **The site tool's prefill for a point in Kosovo** returns XKX instead of + SRB. With it come Southern Europe's composition (food 35.7% instead of + Serbia's 44.3%), a food k of 0.333 instead of 0.143, and Kosovo's population + growth (−0.08% a year instead of −0.76%). + - **Climate TRACE's pipeline is unaffected.** None of its paths calls + `_iso3_for`. + + Tests: `tests/test_country_lookup.py`. + ([#77](https://github.com/RMI/SWEET_python/pull/77), + [RMI/WasteMAP#857](https://github.com/RMI/WasteMAP/pull/857)) + +- **A blank or junk country is not found, instead of being modelled as another + country; the site tool's prefill finds every country the geocoder names** + (model-output change for the prefill in Sint Maarten). `_iso3_for` ends with + pycountry's fuzzy search, which scores every country for an empty name and + scores a country for any of its subdivisions that match: + - **No country was the United Kingdom.** `""` and `" "` resolved to GBR, the + country with the most subdivisions, so a Custom Location or site sent + `country=` was modelled as the UK (30,434 t CH4 in 2040 for 2 million + people). `None` raised `AttributeError`, not `LookupError`, so the callers + answered with a crash rather than "not found". + - **A province's name was its country.** "None" was Italy (Frosinone), "nan" + Thailand (Nan province), "-" France, "0" Sweden, "Sint Maarten" the + Netherlands rather than SXM. + - **The prefill crashed for a point in no country.** Nominatim can't place a + point at sea, so geopy returns None, and its address for Antarctica has no + country. Both raised `AttributeError`. + - **The prefill failed for 11 names Nominatim uses.** Measured over 306 + reverse-geocoded points (one per country and territory): "Turkey", "Ivory + Coast", "Cape Verde", "East Timor", "Democratic Republic of the Congo", + "Congo-Brazzaville", "Palestinian Territories", "Sahrawi Arab Democratic + Republic", "Northern Cyprus", "Somaliland" and "South Ossetia" are spelled + neither SWEET's nor pycountry's way, so each raised "Country '...' not + found." + + `_iso3_for` now raises `LookupError` for anything that isn't a non-blank + string, strips the name, and takes the fuzzy search's answer only when the + name is whole words of that country's own name, or its initials ("UK", + "UAE"). A province's name is no longer found even where the answer was right + ("England"). The prefill (`_iso3_at`) tries the geocoder's English name, then + its ISO 3166-1 code, and says "No country at , ." where there is + none. The name comes first because SWEET has its own codes for Aruba, Curaçao + and Sint Maarten, which Nominatim codes as the Netherlands. + + Effect: + - **Over the 222 country names Nominatim gave**, 210 resolve as before (the + breakaway regions it names as countries resolve through its code to the + country they're in: Abkhazia and South Ossetia to Georgia, Northern Cyprus + to Cyprus, Somaliland to Somalia). The 11 above now resolve. Sint Maarten + moves from NLD to SXM: an open dump rather than a landfill, food 48.2% + rather than 35.0%, growth 0.63% a year rather than 0.15%. + - **Every code and name in SWEET's own table resolves as before**, so the + city tool's Custom Location, which sends a code, is bit-identical for every + country. + - **Climate TRACE's pipeline is unaffected.** None of its paths calls + `_iso3_for`. + + Tests: `tests/test_country_lookup.py`. + ([#80](https://github.com/RMI/SWEET_python/pull/80), + [RMI/WasteMAP#859](https://github.com/RMI/WasteMAP/pull/859)) + +## Known limitations + +- **The three regions fall back to all dumpsite, Brunei apart.** Central Asia + and Rest of Oceania have done the same since #18, for the same reason: the + source gives them no disposal figures. That is coarse for countries such as South Africa, + which landfill most of their waste. The cities table uses the same fallback. + ([#74](https://github.com/RMI/SWEET_python/pull/74)) + +- **Germany and Switzerland now landfill everything they don't compost or + burn.** Their rows file the rest under "unspecified", which holds recycling, + and Custom Location has no recycling default. That overstates their + landfilling, as it already did for Japan, Austria, the Netherlands and Sweden, + whose rows landfill 1–3%. ([#74](https://github.com/RMI/SWEET_python/pull/74)) + +- **Sri Lanka gives the highest figure of the 31: 122,745 t.** Its per-capita + default in `msw_per_capita_country` is 1.86 t/year (5.1 kg/person/day). + Fourteen countries there are above 3 kg/person/day, Kuwait highest at 8.4. + That is a separate data problem, which the zero split had hidden for Sri Lanka. + ([#74](https://github.com/RMI/SWEET_python/pull/74)) diff --git a/changelog/README.md b/changelog/README.md index 98abc33..e1f19ee 100644 --- a/changelog/README.md +++ b/changelog/README.md @@ -7,15 +7,30 @@ emissions. It is the modeling library that the WasteMAP backend API depends on, and this folder tracks notable changes to it. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). -The project does not publish semantic version tags, so releases are tracked by -**calendar month**: each entry file covers a one-month span and is named -`YYYY-MM.md`. +Entries are filed by **calendar month**: each file covers the changes merged in +one month and is named `YYYY-MM.md`. + +## Releases and this changelog + +SWEET's releases are git tags named `sweet/YYYY.MM.N`, cut from `main` by the +Release workflow (see the [README](../README.md#releases)). A release and these +files do different jobs: + +- **A GitHub Release's notes** are generated when the release is cut: the pull + requests merged since the previous release, and a link to that month's file + here. Nobody writes them by hand. +- **The monthly files** keep the human-written entries: what changed, why, and + what it does to the numbers. Write the entry in the pull request that makes the + change, under the month it merges in, whichever release ends up shipping it. + +A release's link points at its month's file as of the release's commit, so it +shows the month up to that release. Entries are not grouped by release. ## Entries Newest first: -- [2026-10](2026-10.md) — A city-level advanced DST site can state its own decay rate (`k_override`) and oxidation rate (`oxidation_override`, used as stated, with null years handed back to the model), and `run_advanced_dst_city_defaults` reports the rates the model uses where a site states none; requests without them are unchanged +- [2026-10](2026-10.md) — Indonesia sizes its waste from its own IPCC per-capita row instead of the South-Eastern Asia default, 2.4× higher (model-output change); a site with no recorded type is guessed by Climate TRACE's rule (income, region and named countries) in every SWEET path, moving 55 countries' unknown sites from open dump to sanitary landfill in the site tool (model-output change); gas collection efficiency and oxidation follow a site's reports year by year on rows that carry their year, and a modelled site keeps its measured efficiency (no output change on current data); `load_csv_new` normalizes a city's composition as the other loaders do, so WasteMAP's city DST runs for São Paulo (99%) and Brasília (87%), whose every scenario it rejected as "Invalid waste fractions"; the city DST of Brazil's precomputed cities scales by 100 ÷ their published sum (Brasília +14.9%, most −0.1%), and the no-change scenario reproduces the baseline (model-output change); a Custom Location city models landfill methane in every country: in 31 (Southern Asia, South-Eastern Asia and Southern Africa, which have no regional disposal row, plus Canada, Germany and Switzerland, whose rows are all zeros) its disposal split was (0, 0, 0), so its landfills took none of its waste; it now gets the cities table's split for a city with no landfill data (model-output change); the City DST's "move waste to existing gas capture" slider changes a city's emissions; it computed a new split and never applied it to the landfills, so every setting returned the no-change scenario for the 7 cities with a live slider (its arithmetic also scaled the shares without capture up, not down) (model-output change); the City DST's "add gas capture" lever applies the oxidation it sets (0.22 for the landfill it fits with gas capture, 0.1 for the dumpsite it converts) instead of resetting both to the no-capture defaults at the end of `implement_dst_changes_simple_v1_5`; its results fall another 10–13% after the implementation year, a city that landfills everything from −60% to −65.3% against no change in 2040 (model-output change); a Custom Location in Kosovo builds: SWEET looks a country up in its own table before asking pycountry, which has no Kosovo, so Kosovo's code XKX no longer raises "Country 'XKX' not found." (WasteMAP answered 500), and the name "Kosovo" no longer resolves to Serbia, which the site tool's prefill for a point in Kosovo returned (model-output change for a country given by name); a blank or junk country is not found instead of being modelled as another country (`""` was the UK, `"None"` Italy), and the site tool's prefill finds the country for a point in Turkey, both Congos and eight more places, and says so for a point in none, where it crashed (model-output change for a point in Sint Maarten); a city-level advanced DST site can state its own decay rate (`k_override`) and oxidation rate (`oxidation_override`, used as stated, with null years handed back to the model), and `run_advanced_dst_city_defaults` reports the rates the model uses where a site states none; requests without them are unchanged - [2026-09](2026-09.md) — A disposal site can accept waste the city did not send it (`accepted_waste_mass`), modelled as a second stream at the same site, with per-site emissions returned split into the city's share and the rest; the city's own total is untouched by construction. WasteMAP's site tool, city tool and Custom Location grow waste with the country's UN population year by year, as Climate TRACE does; big cities keep their own UN city rates (model-output change); model output frames get one fixed waste-type column order instead of a per-process one (a `set` fed the ordering, and Python randomizes set iteration per process), moving values in the last bit only, where a sum runs across the reordered columns; City DST's diversion sum stops passing pandas a keyword pandas is removing, which would have broken the model path on pandas 4 behind a misleading `AttributeError` (no output change); annual model no longer charges a flare's inefficiency twice (a hardcoded 2% slip stacked on the `flaring` destruction efficiency), so it agrees with the monthly model again and a site with gas capture emits ~3.5% less (model-output change); `gas_capture_efficiency` bounded to [0, 1], where an out-of-range value used to produce negative emissions silently; one methane density constant instead of two 9.1% apart - [2026-08](2026-08.md) — Single-site adst gains optional `depth` (deep-dump MCF bump) and `k_override` (caller-supplied decomposition rate) inputs, restoring the last two site-DST levers; `/sdst` flaring efficiency reaches the model again after a variable-name bug silently forced flare destruction to 0.98; annual model applies cover oxidation by emission year not deposit year, fixing biocover having no effect on closed landfills (WasteMAP #719); `City.sdst_v1_5` custom-site path holds the scenario equal to the baseline before the implementation year even when composition changes (was back-dating the new composition onto pre-implementation deposits) (model-output change); MCF consolidated into a new `SWEET_python.mcf` module and both dump types moved to the IPCC uncategorised-SWDS 0.6 (open dumps up from 0.4, controlled dumps down from 0.7), with a supplied depth now selecting the deep/shallow category (model-output change) - [2026-07](2026-07.md) — All ten waste types eligible for combustion (metal/glass/other added); methane-only model treats combustion as landfill diversion (model-output change) diff --git a/requirements.txt b/requirements.txt index 59896e1..83bb2e2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,11 +10,11 @@ annotated-types==0.7.0 # via pydantic anyio==4.14.1 # via starlette -asyncpg==0.31.0 +asyncpg==0.32.0 # via -r requirements.in certifi==2026.7.22 # via -r requirements.in -fastapi==0.141.1 +fastapi==0.143.0 # via -r requirements.in geographiclib==2.1 # via geopy @@ -22,38 +22,41 @@ geopy==2.5.0 # via -r requirements.in idna==3.18 # via anyio -numpy==2.5.3 +numpy==2.5.4 # via # -r requirements.in # pandas +opentelemetry-api==1.45.0 + # via fastapi pandas==3.0.6 # via -r requirements.in psycopg2==2.9.13 # via -r requirements.in pycountry==26.2.16 # via -r requirements.in -pydantic==2.13.5 +pydantic==2.14.0 # via # -r requirements.in # fastapi -pydantic-core==2.46.5 +pydantic-core==2.50.0 # via pydantic python-dateutil==2.9.0.post0 # via pandas six==1.17.0 # via python-dateutil -sqlalchemy==2.1.1 +sqlalchemy==2.1.4 # via -r requirements.in starlette==1.3.1 # via fastapi -typing-extensions==4.15.0 +typing-extensions==4.16.0 # via # fastapi + # opentelemetry-api # pydantic # pydantic-core # sqlalchemy # typing-inspection -typing-inspection==0.4.2 +typing-inspection==0.4.4 # via # fastapi # pydantic diff --git a/scripts/next_release.py b/scripts/next_release.py new file mode 100755 index 0000000..eb7e66a --- /dev/null +++ b/scripts/next_release.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Name SWEET's next release, check a release name, or find the latest one. + +A SWEET release is a git tag ``sweet/YYYY.MM.N``: the UTC year and month it was cut, +then a counter that starts at 0 each month. ``.github/workflows/release.yml`` is the +only caller; it lives here, rather than inline in the workflow, so the arithmetic has +tests (``tests/test_next_release.py``) and no network is needed to run them. + +Tag names come in on stdin, one per line -- whatever ``git tag --list`` printed -- so +this never runs git itself and a test can hand it any list it likes: + + git tag --list 'sweet/*' | scripts/next_release.py # next name this month + git tag --list 'sweet/*' | scripts/next_release.py 2026.10 # next name in 2026-10 + scripts/next_release.py --check sweet/2026.10.3 # is this a release name? + git tag --merged HEAD | scripts/next_release.py --latest # newest release in a list + +Anything on stdin that is not a release name is ignored, not an error: the repository +also carries other projects' tags (``wastemap/2026.10.07``, ``submission/08_03_26``, +``v2026.08.17``), and a hand-pushed ``sweet/test`` must not stop a real release. + +Standard library only, so the workflow can run it with the runner's own ``python3`` +before any dependencies are installed. +""" + +from __future__ import annotations + +import re +import sys +from datetime import datetime, timezone +from typing import Iterable, Optional, Tuple + +# The pattern the workflow documents, with [0-9] rather than \d (\d also matches +# non-ASCII digits) and no leading zeros on N, so each release has exactly one +# spelling: sweet/2026.10.1 and sweet/2026.10.01 cannot both exist. +RELEASE_NAME = re.compile(r"sweet/([0-9]{4})\.([0-9]{2})\.(0|[1-9][0-9]*)") +MONTH = re.compile(r"([0-9]{4})\.([0-9]{2})") + + +def parse(name: str) -> Optional[Tuple[int, int, int]]: + """``(year, month, n)`` for a release name, or ``None`` for anything else. + + The month must be 01-12 as well as two digits: ``sweet/2026.13.0`` matches the + documented pattern but can only be a typo, and a release named for a month that + does not exist would sort after every real one. + """ + m = RELEASE_NAME.fullmatch(name.strip()) + if not m: + return None + year, month, n = (int(g) for g in m.groups()) + if not 1 <= month <= 12: + return None + return year, month, n + + +def next_release(tags: Iterable[str], month: str) -> str: + """The next free release name in ``month`` (``YYYY.MM``), given existing tags. + + One more than the highest N already used that month, not the count of names: + if a gap ever exists (someone named ``.3`` by hand after ``.0`` and ``.1``), a + count would hand out ``.3`` again, and a release must always sort after the ones + before it. + """ + m = MONTH.fullmatch(month.strip()) + if not m or not 1 <= int(m.group(2)) <= 12: + raise ValueError(f"month must be YYYY.MM, got {month!r}") + year, mon = int(m.group(1)), int(m.group(2)) + used = [p[2] for p in map(parse, tags) if p and p[:2] == (year, mon)] + n = max(used) + 1 if used else 0 + return f"sweet/{year:04d}.{mon:02d}.{n}" + + +def latest_release(tags: Iterable[str]) -> Optional[str]: + """The highest release name in ``tags``, by (year, month, N), or ``None``. + + Compared as numbers, not strings: as text, ``sweet/2026.10.10`` sorts before + ``sweet/2026.10.9``. + """ + releases = [(p, t.strip()) for t in tags if (p := parse(t))] + return max(releases)[1] if releases else None + + +def _current_month() -> str: + return datetime.now(timezone.utc).strftime("%Y.%m") + + +def main(argv: list[str]) -> int: + if argv[:1] == ["--check"]: + if len(argv) != 2: + print("usage: next_release.py --check NAME", file=sys.stderr) + return 2 + name = argv[1].strip() + if parse(name) is None: + print( + f"{argv[1]!r} is not a release name: expected sweet/YYYY.MM.N, " + "with a real month and no leading zero on N (e.g. sweet/2026.10.0)", + file=sys.stderr, + ) + return 1 + # Printed back trimmed, so a name pasted with a stray space is used as the + # tag it was meant to be rather than refused for the space. + print(name) + return 0 + + tags = sys.stdin.read().splitlines() + + if argv[:1] == ["--latest"]: + latest = latest_release(tags) + if latest: + print(latest) + return 0 + + if len(argv) > 1: + print("usage: next_release.py [YYYY.MM]", file=sys.stderr) + return 2 + try: + print(next_release(tags, argv[0] if argv else _current_month())) + except ValueError as exc: + print(exc, file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/setup.py b/setup.py index 45eacaa..4d99381 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,53 @@ import pathlib +import subprocess + from setuptools import setup, find_packages +from setuptools.command.build_py import build_py here = pathlib.Path(__file__).parent + +def _clean_head(repo: pathlib.Path): + """The commit *repo* is checked out at, or None if it is not a clean git checkout. + + A build from a dirty tree is not that commit, so it gets no stamp rather than a + wrong one. + """ + def git(*args): + return subprocess.run( + ["git", "-C", str(repo), *args], capture_output=True, text=True, timeout=30, check=True + ).stdout.strip() + + try: + sha = git("rev-parse", "HEAD") + dirty = git("status", "--porcelain", "--untracked-files=no") + except Exception: + return None + return sha if len(sha) == 40 and not dirty else None + + +class build_py_with_commit(build_py): + """Write the commit being built into the built package, as SWEET_python/_build_sha.py. + + pip installs SWEET from git (`pip install git+https://...@sweet/2026.10.0`) by + cloning it and building from that checkout, so the commit is known at build time and + nowhere afterwards: the installed copy has no .git, and only an app that sets + SWEET_GIT_SHA could say which code it got. The stamp goes into the BUILD directory, + never the source tree, so an editable install of a checkout can't pick up a stale one. + """ + + def run(self): + super().run() + sha = _clean_head(here) + if sha and not self.dry_run: + target = pathlib.Path(self.build_lib) / "SWEET_python" / "_build_sha.py" + target.write_text( + '"""Written by setup.py when this copy was built. Not in git."""\n' + f'GIT_SHA = "{sha}"\n' + ) + # Read the abstract (unpinned) dependencies from requirements.in so consumers are # not over-constrained. The pinned lockfile in requirements.txt is generated from # this file with pip-tools (pip-compile) and is used for reproducible installs and @@ -16,14 +59,16 @@ setup( name="SWEET_python", - # Decoration, not a version. SWEET never ships on its own -- it reaches the world - # only inside a Climate TRACE run or a WasteMAP deploy, each of which has an - # identity already -- so there is no hand-maintained number here to keep honest. - # What a caller should read is SWEET_python.__version__, which is the commit this - # copy was installed from. See the package docstring, and VERSIONING.md in - # RMI_Climate_TRACE_Waste_Methane. + # Decoration, not a version. SWEET's releases are git tags named sweet/YYYY.MM.N, + # cut from main by .github/workflows/release.yml (see the README's Releases + # section). This string does not follow them and is never bumped -- pip reports + # 0.1 whichever release it installed. What a caller should read is + # SWEET_python.__version__, which is the commit this copy was installed from; a + # release tag names exactly one commit, so the commit identifies the release too. + # See the package docstring. version="0.1", packages=find_packages(), include_package_data=True, install_requires=requirements, + cmdclass={"build_py": build_py_with_commit}, ) diff --git a/tests/test_add_gas_capture_oxidation.py b/tests/test_add_gas_capture_oxidation.py new file mode 100644 index 0000000..9445680 --- /dev/null +++ b/tests/test_add_gas_capture_oxidation.py @@ -0,0 +1,186 @@ +"""Regression tests for the oxidation the City DST's "add gas capture" lever sets +(``add_gas`` in ``City.implement_dst_changes_simple_v1_5``, generic branch). + +Bug (fixed): after the implementation year, ``add_gas`` gives the landfill +without capture 60% gas capture and an oxidation factor of 0.22, and converts the +dumpsite to a controlled dumpsite with 30% capture and 0.1. It sets +``skip_ox = True`` so ``Landfill.estimate_emissions`` keeps those oxidation +factors. The emissions loop at the end of the method overwrote that with +``skip_ox = scenario_parameters.sites_method``, which is falsy on this branch, so +``estimate_emissions`` reset each oxidation factor to the default for a site +without gas capture: 0.1 for the landfill, 0 for the dumpsite. Gas capture still +applied; the oxidation never did. A city that landfills everything came out 60% +below the no-change scenario after the implementation year instead of 65.3%. + +0.22 and 0.1 are the values ``Landfill.estimate_emissions`` itself gives a +landfill and a controlled dumpsite with gas collection (its ``ox_cap`` table, and +``defaults_2019.oxidation_factor["with_lfg"]``). + +These tests build a city without gas capture from country defaults +(``dst_baseline_blank``, no database), give it a split, and run the real +``implement_dst_changes_simple_v1_5``. +""" + +import copy + +import pandas as pd +import pytest + +from SWEET_python.city_params import City, DiversionFractions +from SWEET_python.class_defs import SplitFractions +from SWEET_python.landfill import Landfill + +_COUNTRY, _POP, _PRECIP, _TEMP = "Mexico", 2_000_000, 600.0, 17.0 +_IMPLEMENT_YEAR, _SCENARIO = 2026, 1 +_BEFORE = slice(None, _IMPLEMENT_YEAR) +_AFTER = slice(_IMPLEMENT_YEAR + 1, None) + +# Each site add_gas converts: (gas capture it adds, oxidation factor before, after). +_LANDFILL_WITHOUT_CAPTURE = (0.6, 0.1, 0.22) +_DUMPSITE = (0.3, 0.0, 0.1) + + +def _city_with_split(with_capture, without_capture, dumpsite) -> City: + """A city whose baseline sends its disposal to the generic split given, as + WasteMAP's ``load_city`` builds one from the cities table.""" + city = City("add_gas_oxidation_regression") + city.dst_baseline_blank(_COUNTRY, _POP, _PRECIP, _TEMP) + baseline = city.baseline_parameters + baseline.split_fractions = SplitFractions( + landfill_w_capture=with_capture, + landfill_wo_capture=without_capture, + dumpsite=dumpsite, + ) + city._calculate_divs() + baseline.repopulate_attr_dicts() + for landfill in baseline.landfills: + landfill.estimate_emissions() + city.estimate_diversion_emissions(scenario=0) + city.sum_landfill_emissions(scenario=0) + return city + + +@pytest.fixture(scope="module") +def city_without_capture() -> City: + """Half its disposal to a landfill without gas capture, half to a dumpsite: a + city the UI offers "add gas capture" to.""" + return _city_with_split(0.0, 0.5, 0.5) + + +def _implement( + city: City, + add_gas: int, + new_gas_pct: float = 0.0, + div_fractions: DiversionFractions = None, + sites_method=None, +): + """Run the city DST on a copy of ``city``, keeping its diversions unless + others are given.""" + city = copy.deepcopy(city) + if div_fractions is None: + div = city.baseline_parameters.div_fractions.iloc[0] + div_fractions = DiversionFractions( + compost=float(div["compost"]), + anaerobic=float(div["anaerobic"]), + combustion=float(div["combustion"]), + recycling=float(div["recycling"]), + ) + if sites_method is not None: + city.baseline_parameters.sites_method = sites_method + city.implement_dst_changes_simple_v1_5( + div_fractions, + add_gas, + 0, + new_gas_pct, + 0.0, + _IMPLEMENT_YEAR, + _SCENARIO, + ) + return city.scenario_parameters[_SCENARIO - 1] + + +@pytest.mark.parametrize("new_gas_pct", [0.0, 0.2]) +@pytest.mark.parametrize( + "index, site", + [(1, _LANDFILL_WITHOUT_CAPTURE), (2, _DUMPSITE)], + ids=["landfill", "dumpsite"], +) +def test_add_gas_keeps_the_oxidation_it_sets(city_without_capture, index, site, new_gas_pct): + """Before the fix the landfill came out at 0.1 and the dumpsite at 0 in every + year. The UI also lets a city add a new landfill with gas capture alongside.""" + _, before, after = site + oxidation = _implement(city_without_capture, 1, new_gas_pct).landfills[index].oxidation_factor + + assert isinstance(oxidation, pd.Series), f"reset to {oxidation!r}" + assert (oxidation.loc[_BEFORE] == before).all() + assert (oxidation.loc[_AFTER] == after).all() + + +@pytest.mark.parametrize( + "index, site", + [(1, _LANDFILL_WITHOUT_CAPTURE), (2, _DUMPSITE)], + ids=["landfill", "dumpsite"], +) +def test_add_gas_emissions_include_the_oxidation(city_without_capture, index, site): + """After the implementation year each converted site emits what it did, less + the gas captured, with the new oxidation in place of the old: 0.4 x 0.78 / 0.9 + for the landfill and 0.7 x 0.9 for the dumpsite. Before the fix the factors + were capture alone, 0.4 and 0.7. Up to the implementation year nothing moves.""" + capture, before, after = site + no_change = _implement(city_without_capture, 0).landfills[index].emissions["total"] + added = _implement(city_without_capture, 1).landfills[index].emissions["total"] + + pd.testing.assert_series_equal(added.loc[_BEFORE], no_change.loc[_BEFORE]) + factor = (1 - capture) * (1 - after) / (1 - before) + pd.testing.assert_series_equal(added.loc[_AFTER], no_change.loc[_AFTER] * factor, rtol=1e-12) + + +def test_a_city_that_landfills_everything_falls_65_percent(): + """No diversion and all disposal to a landfill without capture: add_gas cuts + the city's emissions after the implementation year by 65.3%, not 60%.""" + city = _city_with_split(0.0, 1.0, 0.0) + no_diversion = DiversionFractions(compost=0.0, anaerobic=0.0, combustion=0.0, recycling=0.0) + no_change = _implement(city, 0, div_fractions=no_diversion).total_emissions["total"] + added = _implement(city, 1, div_fractions=no_diversion).total_emissions["total"] + + assert 1 - added.loc[2040] / no_change.loc[2040] == pytest.approx(0.6533, abs=1e-4) + + +def test_without_add_gas_each_site_keeps_its_default_oxidation(city_without_capture): + """The no-change scenario is untouched: every site takes the default for its + type and gas capture, and the landfills emit what the baseline does.""" + scenario = _implement(city_without_capture, 0) + + assert [lf.oxidation_factor for lf in scenario.landfills] == [0.22, 0.1, 0] + for scenario_lf, baseline_lf in zip( + scenario.landfills, city_without_capture.baseline_parameters.landfills + ): + pd.testing.assert_frame_equal(scenario_lf.emissions, baseline_lf.emissions, rtol=1e-12) + + +@pytest.mark.parametrize( + "sites_method, add_gas, skip_ox", + [ + (False, 0, False), + (False, 1, True), + # WasteMAP's Brazilian cities-table rows took this branch with these same + # generic landfills before RMI/WasteMAP#851. Real sites are ``advanced`` + # and keep the oxidation they carry either way. + (True, 0, True), + (True, 1, True), + ], +) +def test_each_branch_passes_its_own_skip_ox( + city_without_capture, monkeypatch, sites_method, add_gas, skip_ox +): + passed = [] + estimate_emissions = Landfill.estimate_emissions + + def recording(self, skip_ox=False, **kwargs): + passed.append(skip_ox) + return estimate_emissions(self, skip_ox=skip_ox, **kwargs) + + monkeypatch.setattr(Landfill, "estimate_emissions", recording) + _implement(city_without_capture, add_gas, sites_method=sites_method) + + assert passed == [skip_ox] * 3 diff --git a/tests/test_carried_series.py b/tests/test_carried_series.py new file mode 100644 index 0000000..e63add4 --- /dev/null +++ b/tests/test_carried_series.py @@ -0,0 +1,127 @@ +"""Gas collection efficiency and oxidation from rows that carry their own year. + +The Climate TRACE pipeline's consolidated input gives a site one row per year and marks +each with `row_year`. For those rows a reported value belongs to its year: one value is +used in every year, and several are carried back and forward (the first back to the +start, each forward to the next report, the latest to the end). Rows without `row_year` +-- the legacy input, and every other caller -- keep the old rule, which +test_gas_capture_gate.py pins. +""" + +import numpy as np +import pandas as pd +import pytest + +from SWEET_python.city_params import ( + GAS_EFF_OPTIONS, + _build_oxidation_series, + _carried_series, + _resolve_gas_capture, +) + +YEARS = range(1970, 2051) + +# Polk County North Central Landfill (GHGRP 1006859), as GHGRP reports it. +POLK = {2010: 0.58, 2011: 0.58, 2012: 0.74, 2013: 0.77, 2014: 0.77, 2015: 0.78, + 2016: 0.78, 2017: 0.78, 2018: 0.67, 2019: 0.67, 2020: 0.67, 2021: 0.67, + 2022: 0.67, 2023: 0.76} + + +def _rows(year_values, column="gas_collection_efficiency", **extra): + frame = pd.DataFrame({"row_year": list(year_values), column: list(year_values.values())}) + for name, value in extra.items(): + frame[name] = value + return frame + + +def test_one_value_is_used_in_every_year(): + series = _carried_series([2015, 2018, 2023], [0.76, 0.76, 0.76], YEARS) + assert (series == 0.76).all() and list(series.index) == list(YEARS) + + +def test_several_values_are_carried_back_and_forward(): + series = _carried_series(list(POLK), list(POLK.values()), YEARS) + assert series.loc[1970] == 0.58 # the first report, carried back + assert series.loc[2011] == 0.58 + assert series.loc[2012] == 0.74 + assert series.loc[2020] == 0.67 + assert series.loc[2023] == 0.76 + assert series.loc[2050] == 0.76 # the latest report, carried forward + + +def test_a_gap_carries_the_last_report_forward(): + series = _carried_series([2010, 2015], [0.5, 0.7], YEARS) + assert series.loc[2014] == 0.5 and series.loc[2015] == 0.7 + + +def test_nothing_reported_is_none(): + assert _carried_series([2010, np.nan], [np.nan, 0.5], YEARS) is None + + +def test_two_values_in_one_year_are_averaged(): + series = _carried_series([2010, 2010, 2012], [0.4, 0.6, 0.8], YEARS) + assert series.loc[2011] == pytest.approx(0.5) + + +def test_a_modelled_site_keeps_its_measured_efficiency(): + """No CH4, so no reported_emissions_year. Keyed on that, the measured 0.82 was + replaced by the sanitary default of 0.6.""" + rows = _rows({2021: 0.82, 2022: 0.82}, reported_emissions_year=np.nan) + _, _, gce = _resolve_gas_capture(True, "Sanitary Landfill", pd.Series(dtype=object), rows, YEARS) + assert (gce == 0.82).all() + assert GAS_EFF_OPTIONS["Sanitary Landfill"] != 0.82 + + +def test_gas_capture_follows_the_reports_year_by_year(): + rows = _rows(POLK, reported_emissions_year=list(POLK)) + _, _, gce = _resolve_gas_capture(False, "Sanitary Landfill", pd.Series(dtype=object), rows, YEARS) + assert gce.loc[1990] == 0.58 and gce.loc[2019] == 0.67 and gce.loc[2040] == 0.76 + + +def test_rows_without_a_year_of_their_own_keep_the_old_rule(): + rows = pd.DataFrame({"reported_emissions_year": [2020, 2021], "gas_collection_efficiency": [0.35, 0.45]}) + _, _, gce = _resolve_gas_capture(True, "Sanitary Landfill", pd.Series(dtype=object), rows, YEARS) + assert gce.loc[1999] == pytest.approx(0.40) # the site mean, as before + + +def test_oxidation_follows_the_same_rule(): + rows = _rows({2013: 0.10, 2020: 0.22}, column="oxidation") + series = _build_oxidation_series(0.1, rows.iloc[0], rows, YEARS) + assert series.loc[1980] == 0.10 and series.loc[2019] == 0.10 and series.loc[2045] == 0.22 + + +def test_oxidation_without_input_keeps_the_default(): + rows = _rows({2013: np.nan}, column="oxidation") + series = _build_oxidation_series(0.05, rows.iloc[0], rows, YEARS) + assert (series == 0.05).all() + + +def test_nullable_columns_are_read(): + years = pd.array([2010, pd.NA, 2012], dtype="Int64") + values = pd.array([0.5, 0.6, pd.NA], dtype="Float64") + assert (_carried_series(years, values, YEARS) == 0.5).all() + + +def test_a_measured_efficiency_on_undated_rows_is_kept(): + """Rows carrying `row_year` whose years are all blank must not fall through to the + emissions-year rule, which drops a modelled site's value for the default.""" + rows = pd.DataFrame({"row_year": [np.nan, np.nan], "gas_collection_efficiency": [0.82, 0.82], + "reported_emissions_year": [np.nan, np.nan]}) + _, _, gce = _resolve_gas_capture(True, "Sanitary Landfill", pd.Series(dtype=object), rows, YEARS) + assert (gce == 0.82).all() + + +def test_undated_rows_without_a_value_get_the_gated_default(): + rows = pd.DataFrame({"row_year": [np.nan], "gas_collection_efficiency": [np.nan], + "reported_emissions_year": [np.nan]}) + _, _, gce = _resolve_gas_capture(True, "Sanitary Landfill", pd.Series(dtype=object), rows, YEARS) + assert (gce == GAS_EFF_OPTIONS["Sanitary Landfill"]).all() + + +def test_undated_oxidation_is_one_value_in_every_year(): + """The legacy rule would overwrite 2020 with that row's 0.1; on these rows the + emissions year does not date the value.""" + rows = pd.DataFrame({"row_year": [np.nan, np.nan], "oxidation": [0.1, 0.3], + "reported_emissions_year": [2020, np.nan]}) + series = _build_oxidation_series(0.05, rows.iloc[0], rows, YEARS) + assert np.allclose(series.to_numpy(), 0.2) diff --git a/tests/test_country_lookup.py b/tests/test_country_lookup.py new file mode 100644 index 0000000..33d4ae9 --- /dev/null +++ b/tests/test_country_lookup.py @@ -0,0 +1,246 @@ +"""A Custom Location in Kosovo builds, and a site in Kosovo is modelled as Kosovo. + +pycountry has no Kosovo. ``_iso3_for`` asked pycountry first, so Kosovo's code XKX, +which SWEET's defaults and WasteMAP's country list both use, raised LookupError: +``City.dst_baseline_blank`` turned it into "Country 'XKX' not found." and WasteMAP's +city tool answered 500. By name it was worse: the fuzzy search resolved "Kosovo" to +Serbia, whose province Kosovo-Metohija matches, so the site tool's prefill for a +point in Kosovo returned Serbia's defaults and code. SWEET's own country table now +comes first. +""" + +import asyncio +from types import SimpleNamespace + +import pandas as pd +import pytest + +from SWEET_python import city_params, defaults_2019 +from SWEET_python.city_params import City +from SWEET_python.class_defs import Variant +from SWEET_python.population import country_population_series + +KOSOVO = country_population_series("XKX") + + +def test_a_custom_location_in_kosovo_builds(): + city = City("*") + city.dst_baseline_blank("XKX", 2_000_000, 500.0, 25.0) + + assert city.baseline_parameters.total_emissions["total"].loc[2040] > 0 + assert city.baseline_parameters.population_series.equals(KOSOVO) + + +def test_a_custom_site_in_kosovo_builds(): + city = City("custom site") + city.cityparams_obj_for_blank_site( + country="XKX", + population=None, + precipitation=500.0, + temperature=10.0, + waste_fractions=Variant(baseline=[0.1] * 10, scenario=[0.1] * 10), + waste_mass_year=Variant[int](baseline=2025, scenario=2025), + ) + + assert city.baseline_parameters.population_series.equals(KOSOVO) + + +def _prefill(monkeypatch, place, latlon): + """The site tool's prefill for ``latlon``, where the geocoder answers ``place``. + + ``place`` is what geopy's ``reverse`` returns: an object with Nominatim's JSON as + ``raw``, or None where Nominatim can't place the point. + """ + monkeypatch.setattr( + city_params, "create_geolocator", lambda: SimpleNamespace(reverse=lambda *a, **k: place) + ) + + def no_database(*args, **kwargs): + raise OSError("no database in this test") + + async def no_async_database(*args, **kwargs): + no_database() + + # The weather lookup falls back to its defaults when the database is unreachable. + monkeypatch.setattr(city_params.socket, "create_connection", no_database) + monkeypatch.setattr(city_params.asyncpg, "connect", no_async_database) + + return asyncio.run( + City("latlon_lookup").sdst_prepopulate( + "db", 5432, "user", "password", "wastemap", "disable", + sites_list=pd.DataFrame(), + latlon=latlon, + ) + ) + + +def test_the_site_tool_prefills_a_point_in_kosovo_as_kosovo(monkeypatch): + """The prefill names the country with the geocoder's English name, "Kosovo".""" + place = SimpleNamespace(raw={"address": {"country": "Kosovo"}}) + + prefill = _prefill(monkeypatch, place, (42.66, 21.17)) # Pristina + + assert prefill["iso3"] == "XKX" + + +@pytest.mark.parametrize( + "given, iso3", + [ + ("XKX", "XKX"), + ("xkx", "XKX"), + ("Kosovo", "XKX"), # was Serbia + ("kosovo", "XKX"), + ("Curacao", "CUW"), # was the Netherlands, whose subdivision Curaçao matches + ("Macau", "MAC"), # these four raised LookupError + ("Pitcairn Islands", "PCN"), + ("Svalbard and Jan Mayen Islands", "SJM"), + ("US Minor Islands", "UMI"), + ("Curaçao", "CUW"), # pycountry still answers for names SWEET spells differently + ("United States", "USA"), + ], +) +def test_sweets_own_country_table_comes_before_pycountry(given, iso3): + assert city_params._iso3_for(given) == iso3 + + +def _resolve(country): + try: + return city_params._iso3_for(country) + except LookupError: + return None + + +def test_every_country_sweet_has_defaults_for_is_found_by_its_code(): + wrong = {code: _resolve(code) for code in defaults_2019.region_lookup_iso3} + assert {code: got for code, got in wrong.items() if got != code} == {} + + +def test_every_country_sweet_names_is_found_by_its_name(): + wrong = {name: _resolve(name) for name in defaults_2019.country_to_iso3} + assert { + name: got for name, got in wrong.items() if got != defaults_2019.country_to_iso3[name] + } == {} + + +def test_a_country_that_does_not_exist_is_still_not_found(): + assert _resolve("ZZZ") is None + assert _resolve("Narnia") is None + with pytest.raises(ValueError, match="Country 'ZZZ' not found"): + City("*").dst_baseline_blank("ZZZ", 2_000_000, 500.0, 25.0) + + +# --- no country, and a name that isn't a country's ---------------------------- +# +# pycountry's fuzzy search scores every country for an empty name, and scores a +# country for any of its subdivisions that match. So "" was the United Kingdom +# (the most subdivisions), "None" Italy (Frosinone) and "nan" Thailand (Nan +# province), and the caller got that country's model without a word. + + +@pytest.mark.parametrize("given", ["", " ", "\t\n", None, float("nan"), 0, b"GBR"]) +def test_no_country_is_not_found(given): + with pytest.raises(LookupError): + city_params._iso3_for(given) + + +@pytest.mark.parametrize( + "given", + [ + "None", # was Italy: its province Frosinone contains "none" + "nan", # was Thailand: Nan province + "-", # was France, whose subdivisions are full of hyphens + "0", # was Sweden + "England", # was right, but a province's name is not a country's + "Kurdistan", # was Iraq + ], +) +def test_a_name_only_a_subdivision_matches_is_not_a_country(given): + with pytest.raises(LookupError): + city_params._iso3_for(given) + + +@pytest.mark.parametrize( + "given, iso3", + [ + (" Kosovo ", "XKX"), # was Serbia: SWEET's table missed the padded name + ("GBR\n", "GBR"), + ("Brunei", "BRN"), # whole words of "Brunei Darussalam" + ("Vatican", "VAT"), # of "Holy See (Vatican City State)" + ("Sint Maarten", "SXM"), # was the Netherlands, through its subdivision Sint Maarten + ("UK", "GBR"), # initials of "United Kingdom" + ("UAE", "ARE"), + ("826", "GBR"), # ISO 3166-1 numeric, an exact match + ], +) +def test_a_country_is_still_found_by_its_own_name(given, iso3): + assert city_params._iso3_for(given) == iso3 + + +def test_a_custom_location_with_no_country_is_not_the_united_kingdom(): + with pytest.raises(ValueError, match="Country '' not found"): + City("*").dst_baseline_blank("", 2_000_000, 500.0, 25.0) + + +def test_a_custom_site_with_no_country_is_not_the_united_kingdom(): + with pytest.raises(ValueError, match="Country ' ' not found"): + City("custom site").cityparams_obj_for_blank_site( + country=" ", + population=None, + precipitation=500.0, + temperature=10.0, + waste_fractions=Variant(baseline=[0.1] * 10, scenario=[0.1] * 10), + waste_mass_year=Variant[int](baseline=2025, scenario=2025), + ) + + +# --- the site tool's prefill --------------------------------------------------- + + +def test_the_prefill_for_a_point_at_sea_is_not_found(monkeypatch): + """Nominatim can't place it, so geopy returns None. That raised AttributeError.""" + with pytest.raises(ValueError, match="No country at 0.0, 0.0"): + _prefill(monkeypatch, None, (0.0, 0.0)) + + +def test_the_prefill_for_a_point_in_no_country_is_not_found(monkeypatch): + """Nominatim places McMurdo Station in Antarctica with no country. That raised AttributeError.""" + place = SimpleNamespace(raw={"address": {"road": "Scott Base Road"}}) + + with pytest.raises(ValueError, match="No country at -77.85, 166.68"): + _prefill(monkeypatch, place, (-77.85, 166.68)) + + +@pytest.mark.parametrize( + "country, code, iso3", + [ + # Nominatim's English names, which SWEET's table and pycountry spell otherwise. + # Each raised "Country '...' not found.", so the prefill failed. + ("Turkey", "tr", "TUR"), + ("Ivory Coast", "ci", "CIV"), + ("Cape Verde", "cv", "CPV"), + ("Democratic Republic of the Congo", "cd", "COD"), + ("Congo-Brazzaville", "cg", "COG"), + ("East Timor", "tl", "TLS"), + ("Palestinian Territories", "ps", "PSE"), + ("Sahrawi Arab Democratic Republic", "eh", "ESH"), + ("Northern Cyprus", "cy", "CYP"), + ("Somaliland", "so", "SOM"), + ("South Ossetia", "ge", "GEO"), + ("Abkhazia", "ge", "GEO"), # was Georgia through its subdivision; now through the code + # The name wins where SWEET has the country's own code and Nominatim the Netherlands'. + ("Aruba", "nl", "ABW"), + ("Curaçao", "nl", "CUW"), + ("Sint Maarten", "nl", "SXM"), + ], +) +def test_the_prefill_falls_back_to_the_geocoders_country_code(monkeypatch, country, code, iso3): + place = SimpleNamespace(raw={"address": {"country": country, "country_code": code}}) + + assert _prefill(monkeypatch, place, (0.0, 0.0))["iso3"] == iso3 + + +def test_the_prefill_names_a_country_it_cannot_find(monkeypatch): + place = SimpleNamespace(raw={"address": {"country": "Narnia"}}) + + with pytest.raises(ValueError, match="Country 'Narnia' not found"): + _prefill(monkeypatch, place, (0.0, 0.0)) diff --git a/tests/test_custom_location_disposal_split.py b/tests/test_custom_location_disposal_split.py new file mode 100644 index 0000000..80c151b --- /dev/null +++ b/tests/test_custom_location_disposal_split.py @@ -0,0 +1,189 @@ +"""A Custom Location buries or dumps its waste in every country. + +`City.dst_baseline_blank` (WasteMAP's Custom Location) looked up a country's +disposal split with `fraction_open_dumped_country.get(iso3, +fraction_open_dumped.get(region, 0))`, and the same for `fraction_landfilled`. +Two gaps in those tables made the split (0, 0, 0) for 31 countries: + +- Southern Asia, South-Eastern Asia and Southern Africa have no regional row. + This is not a label mismatch: every other regional table carries these three + regions under the same names, as `0.0 # np.nan`. None of their 28 countries + here has a row of its own. +- Canada's, Germany's and Switzerland's country rows are all zeros. + +With a zero split every landfill took 0% of the waste, so the city modelled no +landfill methane at all. Pakistan and the Philippines at 2M people gave 0 t CH4 +in 2040, against 28,745 t for Mexico. The `except KeyError` fallback written for +this never ran, because `.get` never raises. + +The cities table (`City.load_andre_params`) handles both gaps. A region with no +row falls back to all landfill or all dumpsite by Climate TRACE's site-type rule +(`site_type_defaults.landfill_is_default`), and the three zero-row countries are +landfilled. Custom Location now takes the split the cities table gives a city +with no landfill data (`defaults_2019.disposal_split_for`). +""" + +import numpy as np +import pytest + +from SWEET_python import defaults_2019 +from SWEET_python.city_params import City + +POPULATION = 2_000_000 +PRECIPITATION = 500.0 +TEMPERATURE = 25.0 +YEAR = 2040 + +# Every country whose Custom Location split was (0, 0, 0). +NO_ROW = { + "Southern Asia": ["BGD", "BTN", "IND", "IOT", "IRN", "LKA", "MDV", "NPL", "PAK"], + "South-Eastern Asia": [ + "BRN", "IDN", "KHM", "LAO", "MMR", "MYS", "PHL", "THA", "TLS", "VNM", + ], + "Southern Africa": ["ATF", "BWA", "LSO", "MOZ", "NAM", "SHN", "SWZ", "ZAF", "ZWE"], +} +ZERO_COUNTRY_ROW = ["CAN", "CHE", "DEU"] +ZEROED = sorted([iso3 for codes in NO_ROW.values() for iso3 in codes] + ZERO_COUNTRY_ROW) + + +def _custom_location(iso3): + city = City("Custom Location") + city.dst_baseline_blank(iso3, POPULATION, PRECIPITATION, TEMPERATURE) + return city.baseline_parameters + + +class _NoDataRow(dict): + """A cities-table row in which every field not given is missing (NaN).""" + + def __missing__(self, key): + return np.nan + + +def _cities_table_split(iso3): + """The split `load_andre_params` gives a city with no landfill data in `iso3`. + + The row says where the city is and nothing about its waste: every landfill and + diversion share is NaN, so the loader takes its defaults for both, which is + the cities-table counterpart of a Custom Location. + """ + country = next( + name + for name, code in defaults_2019.country_to_iso3.items() + if code == iso3 and name in defaults_2019.region_lookup + ) + row = _NoDataRow( + { + "country": country, + "iso": iso3, + "population_count": POPULATION, + "population_year": 2022.0, + "population_data_source": "test", + "msw_generated_metric_tons_per_year": 500_000.0, + "msw_generated_year": 2022.0, + "mean_yearly_precip_2000_2021": PRECIPITATION, + "mean_yearly_temp_2000_2021": TEMPERATURE, + "Temperature (C)": TEMPERATURE, + "latitude": 0.0, + "longitude": 0.0, + "historic_growth_rate": 1.0, + "future_growth_rate": 1.0, + } + ) + city = City("Cities table") + city.load_andre_params(row) + return city.baseline_parameters.split_fractions + + +def _how_the_split_is_decided(iso3, region): + if iso3 in defaults_2019.fraction_open_dumped_country: + row = ( + defaults_2019.fraction_open_dumped_country[iso3] + + defaults_2019.fraction_landfilled_country[iso3] + ) + return "country row" if row > 0 else "all-zero country row" + if region in defaults_2019.fraction_open_dumped: + return "regional row" + return "no row" + + +def _one_country_per_region_and_rule(): + """The first country of each (region, which row decides) pair, plus Germany. + + Comparing every country with the cities table takes ten seconds; one per + pair runs every branch of both lookups for every region in a fifth of that. + Germany is added so all three zero-row countries are covered. + """ + first = {} + for iso3, region in sorted(defaults_2019.region_lookup_iso3.items()): + first.setdefault((region, _how_the_split_is_decided(iso3, region)), iso3) + return sorted(set(first.values()) | set(ZERO_COUNTRY_ROW)) + + +@pytest.mark.parametrize("iso3", ZEROED) +def test_a_custom_location_models_methane_from_the_waste_it_disposes_of(iso3): + parameters = _custom_location(iso3) + + assert sum(landfill.fraction_of_waste for landfill in parameters.landfills) == ( + pytest.approx(1.0) + ) + # Landfill methane only. Germany's total was already 26 t from composting + # while its landfills took nothing. + assert parameters.landfill_emissions.loc[YEAR, "total"] > 0 + + +@pytest.mark.parametrize("iso3", ["PAK", "PHL", "ZAF"]) +def test_a_country_in_a_region_with_no_row_dumps_its_waste(iso3): + # None of the three regions makes a country a landfill one, and none of these + # three is high income. + split = _custom_location(iso3).split_fractions + + assert (split.landfill_w_capture, split.landfill_wo_capture, split.dumpsite) == ( + 0.0, + 0.0, + 1.0, + ) + + +def test_a_high_income_country_in_a_region_with_no_row_landfills_its_waste(): + # Brunei, by Climate TRACE's site-type rule, as its cities-table rows do. + split = _custom_location("BRN").split_fractions + + assert (split.landfill_w_capture, split.landfill_wo_capture, split.dumpsite) == ( + 0.0, + 1.0, + 0.0, + ) + + +@pytest.mark.parametrize("iso3", ZERO_COUNTRY_ROW) +def test_a_country_whose_row_is_all_zeros_landfills_its_waste(iso3): + split = _custom_location(iso3).split_fractions + + assert (split.landfill_w_capture, split.landfill_wo_capture, split.dumpsite) == ( + 0.0, + 1.0, + 0.0, + ) + + +@pytest.mark.parametrize("iso3", _one_country_per_region_and_rule()) +def test_a_custom_location_starts_from_the_cities_tables_split(iso3): + custom = _custom_location(iso3).split_fractions + table = _cities_table_split(iso3) + + assert custom.model_dump() == pytest.approx(table.model_dump()) + + +def test_every_country_has_a_disposal_split(): + splits = { + iso3: defaults_2019.disposal_split_for(iso3, region) + for iso3, region in defaults_2019.region_lookup_iso3.items() + } + + assert { + iso3: split + for iso3, split in splits.items() + if set(split) != {"landfill_w_capture", "landfill_wo_capture", "dumpsite"} + or min(split.values()) < 0 + or sum(split.values()) != pytest.approx(1.0) + } == {} diff --git a/tests/test_load_csv_composition.py b/tests/test_load_csv_composition.py new file mode 100644 index 0000000..dcc5990 --- /dev/null +++ b/tests/test_load_csv_composition.py @@ -0,0 +1,181 @@ +"""`City.load_csv_new` has to model a composition that adds up to 100%. + +WasteMAP's city DST loads a city from its published cities table with +`load_csv_new`. Brazil's precomputed rows carry their source composition as it +is: São Paulo's ten `Waste Components` sum to 99%, Brasília's to 87%, and the +national default the other ~4,960 Brazilian municipalities get sums to 100.1%. +Every other loader here normalizes a composition; `load_csv_new` did not, so: + +- `implement_dst_changes_simple_v1_5` rejected São Paulo and Brasília outright. + Its `|sum - 1| > 1e-2` check fails Brasília by 13 points and São Paulo by + float rounding (1 - 0.99 = 0.010000000000000009), so every scenario request + for Brazil's two largest cities was a 400 "Invalid waste fractions". +- The other Brazilian cities ran, but their typed masses missed or exceeded + the city's waste, and the no-change scenario drifted from the baseline. +""" + +import pandas as pd +import pytest + +from SWEET_python.city_params import City, DiversionFractions +from SWEET_python.constants import WASTE_TYPES + +IMPLEMENT_YEAR = 2026 + +LABELS = { + "food": "Food", + "green": "Green", + "wood": "Wood", + "paper_cardboard": "Paper and Cardboard", + "textiles": "Textiles", + "plastic": "Plastic", + "metal": "Metal", + "glass": "Glass", + "rubber": "Rubber/Leather", + "other": "Other", +} + +# The waste types each diversion takes, for the columns `load_csv_new` reads. +# `make_cities_table` derives a city's diversion components as its composition's +# shares within each set, which reproduces the published values. +DIVERTED = { + ("Composted", "Composted"): ["food", "green", "wood", "paper_cardboard"], + ("Anaerobically Digested", "Digested"): ["food", "green", "wood", "paper_cardboard"], + ("Incinerated", "Incinerated"): [ + "food", "green", "wood", "paper_cardboard", "textiles", "plastic", "rubber", + ], + ("Recycled", "Recycled"): [ + "wood", "paper_cardboard", "textiles", "plastic", "metal", "glass", "rubber", "other", + ], +} + +# Compositions as the June 2026 cities table publishes them, in percent. +SAO_PAULO = {"food": 47.0, "paper_cardboard": 16.0, "plastic": 10.0, "metal": 2.0, + "glass": 1.0, "rubber": 1.0, "other": 22.0} # 99% +BRASILIA = {"food": 53.0, "paper_cardboard": 6.0, "plastic": 5.0, "metal": 1.0, + "glass": 4.0, "other": 18.0} # 87% +BRAZIL_DEFAULT = {"food": 53.5, "paper_cardboard": 17.6, "plastic": 17.5, "metal": 2.2, + "glass": 4.0, "other": 5.3} # 100.1%, e.g. Salvador + +# São Paulo's and Brasília's other inputs, as published. +CITIES = { + "São Paulo": dict( + composition=SAO_PAULO, population=10_000_000.0, waste=4_700_000.0, + historic=3.26, future=0.70, precip=1417.67, temperature=19.39, + compost=3.0, recycling=0.0, landfill=100.0, dumpsite=0.0, + ), + "Brasília": dict( + composition=BRASILIA, population=4_800_000.0, waste=1_199_862.0, + historic=4.47, future=0.60, precip=1416.11, temperature=21.64, + compost=18.2, recycling=23.1, landfill=50.0, dumpsite=50.0, + ), +} + + +def _cities_table_row(name, composition, *, population, waste, historic, future, + precip, temperature, compost, recycling, landfill, dumpsite): + """One city-year of WasteMAP's cities table, cut to the columns `load_csv_new` reads.""" + percent = {waste_type: composition.get(waste_type, 0.0) for waste_type in WASTE_TYPES} + row = { + "City": name, + "Country ISO3": "BRA", + # Written as "True", as make_cities_table does for every Brazil row. + "Uses Sites Method": True, + "Population": population, + "Year of Data Collection (Population)": 2022, + "Population Growth Rate: Historic (%)": historic, + "Population Growth Rate: Future (%)": future, + "Average Annual Precipitation (mm/year)": precip, + "Precipitation Zone": "Moderately Wet", + "Temperature (C)": temperature, + "Waste Generation Rate (tons/year)": waste, + "Waste Generation Rate per Capita (kg/person/day)": waste / population * 1000 / 365, + "Data Source (Waste Mass)": "SINIR 2014", + "MEF: Compost": 0.004243855, + "Methane Capture Efficiency (%)": 0.0, + "Percent of Waste to Landfills with Gas Capture (%)": 0.0, + "Percent of Waste to Landfills without Gas Capture (%)": landfill, + "Percent of Waste to Dumpsites (%)": dumpsite, + "Diversions: Compost (%)": compost, + "Diversions: Anaerobic Digestion (%)": 0.0, + "Diversions: Incineration (%)": 0.0, + "Diversions: Recycling (%)": recycling, + } + row.update({f"Waste Components: {LABELS[w]} (%)": percent[w] for w in WASTE_TYPES}) + for (verb, total), taken in DIVERTED.items(): + pool = sum(percent[w] for w in taken) + row.update( + { + f"Diversion Components: {verb} {LABELS[w]} (% of Total {total})": percent[w] / pool * 100 + for w in taken + } + ) + return pd.DataFrame([row]) + + +def _load(name, composition=None): + """What WasteMAP's `load_city` does with a cities-table row.""" + inputs = dict(CITIES.get(name, CITIES["São Paulo"])) + if composition is not None: + inputs["composition"] = composition + city = City(name) + city.load_csv_new(_cities_table_row(name, **inputs), dst=True) + city._calculate_divs() + for landfill in city.baseline_parameters.landfills: + landfill.estimate_emissions() + city.estimate_diversion_emissions(scenario=0) + city.sum_landfill_emissions(scenario=0) + return city + + +CASES = pytest.mark.parametrize( + "name, composition", + [ + ("São Paulo", SAO_PAULO), + ("Brasília", BRASILIA), + # São Paulo's inputs with the default most Brazilian cities publish. + ("a city on Brazil's default", BRAZIL_DEFAULT), + ], + ids=["sao-paulo-99pct", "brasilia-87pct", "brazil-default-100.1pct"], +) + + +@CASES +def test_the_composition_is_normalized(name, composition): + city = _load(name, composition) + fractions = city.baseline_parameters.waste_fractions.iloc[0] + published = pd.Series(composition).reindex(fractions.index, fill_value=0.0) + + # Each type keeps its published share of the reported total. + pd.testing.assert_series_equal( + fractions, published / published.sum(), check_names=False, rtol=1e-12 + ) + # So the typed masses add up to the city's waste: none goes missing. + masses = city.baseline_parameters.waste_masses.model_dump() + inputs = CITIES.get(name, CITIES["São Paulo"]) + assert sum(masses.values()) == pytest.approx(inputs["waste"], rel=1e-12) + + +@CASES +def test_no_change_reproduces_the_baseline(name, composition): + city = _load(name, composition) + baseline = city.baseline_parameters.total_emissions["total"].copy() + unchanged = city.baseline_parameters.div_fractions.iloc[0] + + # Raised "Invalid waste fractions" for São Paulo and Brasília; drifted + # from the baseline for the default composition. + city.implement_dst_changes_simple_v1_5( + DiversionFractions(**unchanged.to_dict()), 0, 0, 0.0, 0.0, IMPLEMENT_YEAR, 1 + ) + scenario = city.scenario_parameters[0].total_emissions["total"] + pd.testing.assert_series_equal(scenario, baseline, rtol=1e-9) + + +def test_a_composition_that_adds_up_loads_unchanged(): + # Every non-Brazil row of the cities table already sums to 100%, so + # normalizing must leave its fractions exactly as published. + adds_up = {"food": 50.0, "paper_cardboard": 25.0, "plastic": 12.5, "other": 12.5} + city = _load("São Paulo", adds_up) + fractions = city.baseline_parameters.waste_fractions.iloc[0] + published = pd.Series(adds_up).reindex(fractions.index, fill_value=0.0) / 100 + assert (fractions == published).all() diff --git a/tests/test_move_to_existing_gas_capture.py b/tests/test_move_to_existing_gas_capture.py new file mode 100644 index 0000000..9c60872 --- /dev/null +++ b/tests/test_move_to_existing_gas_capture.py @@ -0,0 +1,252 @@ +"""Regression tests for the City DST's "move waste to existing gas capture" lever +(``move_gas`` and ``existing_gas_pct`` in ``City.implement_dst_changes_simple_v1_5``). + +Bug (fixed): for a city on the generic landfill split, the lever rewrote +``split_fractions`` but never passed the new split to the landfills. Each +landfill's waste comes from its own ``fraction_of_waste``, which only the +"new landfill with gas capture" block updated, and the frontend never sends that +lever together with this one. So the slider changed nothing: Lahore (34% of its +waste to gas capture) moved to 90% returned exactly the no-change scenario, as did +every other city that has a live slider. + +The arithmetic was wrong too. It set the capture share and then multiplied the +shares without capture by ``existing / original``, scaling them up: Lahore's +landfill without capture came out at 171% of the city's waste, Mexico City's at +3,848%. With no capture to start from, it zeroed them. Combined with a new +landfill with gas capture, the inflated split failed its sum check and raised a +``TypeError`` (``CustomError`` was built with one argument), which the API +returned as a 500. + +The fix gives the capture share ``existing_gas_pct`` and the landfill and dumpsite +without capture the rest, in their original proportions, then applies that split +to the landfills. + +These tests build a city with gas capture from country defaults +(``dst_baseline_blank``, no database), give it a split, and run the real +``implement_dst_changes_simple_v1_5``. +""" + +import copy +import math + +import pandas as pd +import pytest + +from SWEET_python.city_params import City, CustomError, DiversionFractions +from SWEET_python.class_defs import SplitFractions + +_COUNTRY, _POP, _PRECIP, _TEMP = "Mexico", 2_000_000, 600.0, 17.0 +_IMPLEMENT_YEAR, _SCENARIO = 2026, 1 +# 30% of disposal to a landfill with gas capture, 50% to one without, 20% to a +# dumpsite, so the move has two sites without capture to scale. +_WITH_CAPTURE, _WITHOUT_CAPTURE, _DUMPSITE = 0.3, 0.5, 0.2 + + +def _city_with_split(with_capture, without_capture, dumpsite) -> City: + """A city whose baseline sends its disposal to the generic split given. + + ``dst_baseline_blank`` builds every Custom Location with no gas capture, so the + split is replaced and the landfills rebuilt from it, which is what WasteMAP's + ``load_city`` does after reading a city's split from the cities table. + """ + city = City("move_gas_regression") + city.dst_baseline_blank(_COUNTRY, _POP, _PRECIP, _TEMP) + baseline = city.baseline_parameters + baseline.split_fractions = SplitFractions( + landfill_w_capture=with_capture, + landfill_wo_capture=without_capture, + dumpsite=dumpsite, + ) + city._calculate_divs() + baseline.repopulate_attr_dicts() + for landfill in baseline.landfills: + landfill.estimate_emissions() + city.estimate_diversion_emissions(scenario=0) + city.sum_landfill_emissions(scenario=0) + return city + + +@pytest.fixture(scope="module") +def city_with_capture() -> City: + return _city_with_split(_WITH_CAPTURE, _WITHOUT_CAPTURE, _DUMPSITE) + + +def _implement(city: City, move_gas: int, existing_gas_pct: float, new_gas_pct: float = 0.0): + """Run the city DST on a copy of ``city``, keeping its diversions.""" + city = copy.deepcopy(city) + div = city.baseline_parameters.div_fractions.iloc[0] + div_fractions = DiversionFractions( + compost=float(div["compost"]), + anaerobic=float(div["anaerobic"]), + combustion=float(div["combustion"]), + recycling=float(div["recycling"]), + ) + city.implement_dst_changes_simple_v1_5( + div_fractions, + 0, + move_gas, + new_gas_pct, + existing_gas_pct, + _IMPLEMENT_YEAR, + _SCENARIO, + ) + return city.scenario_parameters[_SCENARIO - 1] + + +def _total(scenario_parameters) -> pd.Series: + return scenario_parameters.total_emissions["total"] + + +def _moved_split(existing_gas_pct): + rest = 1 - existing_gas_pct + without_capture_total = _WITHOUT_CAPTURE + _DUMPSITE + return ( + existing_gas_pct, + _WITHOUT_CAPTURE * rest / without_capture_total, + _DUMPSITE * rest / without_capture_total, + ) + + +def test_moving_waste_to_gas_capture_lowers_emissions(city_with_capture): + """Before the fix this returned the no-change scenario exactly.""" + no_change = _total(_implement(city_with_capture, 0, 0.0)) + moved = _total(_implement(city_with_capture, 1, 0.9)) + + pd.testing.assert_series_equal( + moved.loc[: _IMPLEMENT_YEAR - 1], no_change.loc[: _IMPLEMENT_YEAR - 1] + ) + after = slice(_IMPLEMENT_YEAR + 1, None) + assert (moved.loc[after] < no_change.loc[after]).all(), ( + "moving waste to gas capture did not lower emissions after " + f"{_IMPLEMENT_YEAR}: 2040 {moved.loc[2040]:.1f} vs {no_change.loc[2040]:.1f}" + ) + + +def test_moving_waste_off_gas_capture_raises_emissions(city_with_capture): + no_change = _total(_implement(city_with_capture, 0, 0.0)) + moved = _total(_implement(city_with_capture, 1, 0.0)) + + after = slice(_IMPLEMENT_YEAR + 1, None) + assert (moved.loc[after] > no_change.loc[after]).all(), ( + "moving all waste off gas capture did not raise emissions after " + f"{_IMPLEMENT_YEAR}: 2040 {moved.loc[2040]:.1f} vs {no_change.loc[2040]:.1f}" + ) + + +@pytest.mark.parametrize("existing_gas_pct", [0.0, 0.6, 0.9, 1.0]) +def test_landfills_receive_the_moved_split(city_with_capture, existing_gas_pct): + """The landfill and dumpsite without capture share what capture does not + take, in their original 5:2 proportion, and the split still sums to 1.""" + scenario = _implement(city_with_capture, 1, existing_gas_pct) + expected = _moved_split(existing_gas_pct) + + split = scenario.split_fractions + assert (split.landfill_w_capture, split.landfill_wo_capture, split.dumpsite) == pytest.approx( + expected, abs=1e-12 + ) + fractions = [landfill.fraction_of_waste for landfill in scenario.landfills] + assert fractions == pytest.approx(list(expected), abs=1e-12) + assert sum(fractions) == pytest.approx(1.0, abs=1e-12) + + +def test_moved_waste_is_reallocated_not_created_or_lost(city_with_capture): + """From the implementation year each landfill's waste is the no-change + scenario's, scaled by its new share over its old one, and the total + landfilled each year is unchanged. Before it, nothing moves.""" + no_change = _implement(city_with_capture, 0, 0.0) + moved = _implement(city_with_capture, 1, 0.9) + + after = slice(_IMPLEMENT_YEAR, None) + before = slice(None, _IMPLEMENT_YEAR - 1) + old_split = (_WITH_CAPTURE, _WITHOUT_CAPTURE, _DUMPSITE) + for old_share, new_share, base_lf, moved_lf in zip( + old_split, _moved_split(0.9), no_change.landfills, moved.landfills + ): + pd.testing.assert_frame_equal( + moved_lf.waste_mass_df.loc[before], base_lf.waste_mass_df.loc[before] + ) + pd.testing.assert_frame_equal( + moved_lf.waste_mass_df.loc[after], + base_lf.waste_mass_df.loc[after] * (new_share / old_share), + rtol=1e-12, + ) + + def landfilled(scenario): + return sum(lf.waste_mass_df.sum(axis=1) for lf in scenario.landfills) + + pd.testing.assert_series_equal(landfilled(moved), landfilled(no_change), rtol=1e-12) + + +def test_the_city_own_share_is_the_no_change_scenario(city_with_capture): + no_change = _total(_implement(city_with_capture, 0, 0.0)) + unmoved = _total(_implement(city_with_capture, 1, _WITH_CAPTURE)) + + pd.testing.assert_series_equal(unmoved, no_change, rtol=1e-12) + + +def test_moving_and_adding_a_new_landfill_with_gas_capture(city_with_capture): + """The new landfill takes its share off the top of the moved split. The UI + never sends both levers, but the API accepts them; before the fix the + inflated split failed the sum check with a TypeError (an API 500).""" + scenario = _implement(city_with_capture, 1, 0.9, new_gas_pct=0.2) + + expected = [share * 0.8 for share in _moved_split(0.9)] + [0.2] + fractions = [landfill.fraction_of_waste for landfill in scenario.landfills] + assert fractions == pytest.approx(expected, abs=1e-12) + + +@pytest.mark.parametrize("existing_gas_pct", [-0.1, 1.1, math.nan]) +def test_a_share_outside_zero_to_one_is_rejected(city_with_capture, existing_gas_pct): + with pytest.raises(CustomError) as excinfo: + _implement(city_with_capture, 1, existing_gas_pct) + assert excinfo.value.code == "INVALID_PARAMETERS" + + +@pytest.mark.parametrize( + "split", + [ + (0.0, 0.7, 0.3), + # No disposal at all, the split a Custom Location had in 31 countries + # before RMI/SWEET_python#74. + (0.0, 0.0, 0.0), + ], +) +def test_a_city_without_gas_capture_has_no_landfill_to_move_waste_to(split): + """The UI only offers the lever to a city with gas capture. Before the fix + the API returned the no-change scenario, as if the move had been made. A + share of 0 asks for no change, which is what it gets.""" + city = _city_with_split(*split) + with pytest.raises(CustomError) as excinfo: + _implement(city, 1, 0.5) + assert excinfo.value.code == "INVALID_PARAMETERS" + + pd.testing.assert_series_equal( + _total(_implement(city, 1, 0.0)), _total(_implement(city, 0, 0.0)), rtol=1e-12 + ) + + +def test_a_city_all_to_gas_capture_has_no_site_to_move_waste_back_to(): + """41 cities in the June 2026 cities table send all their disposal to gas + capture. The UI disables the slider for them and sends 100%.""" + city = _city_with_split(1.0, 0.0, 0.0) + with pytest.raises(CustomError) as excinfo: + _implement(city, 1, 0.5) + assert excinfo.value.code == "INVALID_PARAMETERS" + + pd.testing.assert_series_equal( + _total(_implement(city, 1, 1.0)), _total(_implement(city, 0, 0.0)), rtol=1e-12 + ) + + +def test_lahore_moved_to_ninety_percent(): + """Lahore's split in the June 2026 cities table. Before the fix the split came + out as 90% / 171% / 0% and never reached the landfills.""" + city = _city_with_split(0.34430946, 0.65569054, 0.0) + scenario = _implement(city, 1, 0.9) + + split = scenario.split_fractions + assert (split.landfill_w_capture, split.landfill_wo_capture, split.dumpsite) == pytest.approx( + (0.9, 0.1, 0.0), abs=1e-12 + ) + fractions = [landfill.fraction_of_waste for landfill in scenario.landfills] + assert fractions == pytest.approx([0.9, 0.1, 0.0], abs=1e-12) diff --git a/tests/test_msw_per_capita.py b/tests/test_msw_per_capita.py new file mode 100644 index 0000000..390e370 --- /dev/null +++ b/tests/test_msw_per_capita.py @@ -0,0 +1,53 @@ +"""Per-capita waste generation defaults (`defaults_2019.msw_per_capita_country`). + +The table is IPCC 2019 Refinement, Vol. 5 Ch. 2, Table 2A.1 (Updated), 2010 +column, in tonnes per person per year, converted to kg per person per day. Every +row matched the IPCC's except one: Indonesia, which the IPCC lists at 0.19, had +no row. A Custom Location or a cities-table city in Indonesia with no waste +figure of its own was sized from the South-Eastern Asia default instead, 0.46, +2.4 times the country's own row. +""" + +import numpy as np +import pytest + +from SWEET_python import defaults_2019 +from SWEET_python.city_params import City + +_POP = 1_000_000 + + +def _kg_per_day(tonnes_per_year): + return tonnes_per_year / 365 * 1000 + + +def test_indonesia_has_its_own_row(): + assert defaults_2019.msw_per_capita_country["IDN"] == pytest.approx(_kg_per_day(0.19)) + + +def test_every_south_eastern_asian_country_the_ipcc_lists_has_a_row(): + # Table 2A.1's South-Eastern Asia block. Cambodia and Timor-Leste are not in + # it, so they take the regional default. + ipcc_rows = { + "BRN": 0.32, "IDN": 0.19, "LAO": 0.26, "MYS": 0.55, "MMR": 0.16, + "PHL": 0.18, "SGP": 1.28, "THA": 0.64, "VNM": 0.53, + } + for iso3, tonnes_per_year in ipcc_rows.items(): + assert defaults_2019.msw_per_capita_country[iso3] == pytest.approx( + _kg_per_day(tonnes_per_year) + ), iso3 + + +def test_a_custom_location_in_indonesia_is_sized_from_indonesias_row(): + city = City("custom") + city.dst_baseline_blank("Indonesia", _POP, 2000.0, 27.0) + + # Was 460,000: the South-Eastern Asia default, 0.46 t per person per year. + assert np.allclose(city.baseline_parameters.waste_mass, 0.19 * _POP) + + +def test_a_country_without_a_row_still_takes_its_regions_default(): + city = City("custom") + city.dst_baseline_blank("Cambodia", _POP, 2000.0, 27.0) + + assert np.allclose(city.baseline_parameters.waste_mass, 0.46 * _POP) diff --git a/tests/test_next_release.py b/tests/test_next_release.py new file mode 100644 index 0000000..9eb726c --- /dev/null +++ b/tests/test_next_release.py @@ -0,0 +1,184 @@ +"""``scripts/next_release.py`` names SWEET's releases; these hold its arithmetic. + +The release workflow pipes ``git tag --list`` into the script and tags whatever comes +back, and a release tag is never moved or reused -- so a wrong name here is a wrong +name forever. Every test hands the script a tag list directly: no git, no network. +""" + +from __future__ import annotations + +import importlib.util +import subprocess +import sys +from pathlib import Path + +import pytest + +SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "next_release.py" + +# Loaded by path, not imported as ``scripts.next_release``: ``scripts/`` is not a +# package, and in an environment that also has WasteMAP installed editable, the name +# ``scripts`` already belongs to WasteMAP. +_spec = importlib.util.spec_from_file_location("next_release", SCRIPT) +next_release_mod = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(next_release_mod) + +next_release = next_release_mod.next_release +latest_release = next_release_mod.latest_release + +# Tags this repository really carries that belong to other projects. +OTHER_PROJECTS = [ + "v2026.08.17", + "wastemap/2026.08.17", + "wastemap/2026.10.07", + "submission/08_03_26", +] + + +def test_first_release_of_a_month_is_zero(): + assert next_release([], "2026.10") == "sweet/2026.10.0" + assert next_release(OTHER_PROJECTS, "2026.10") == "sweet/2026.10.0" + + +def test_after_zero_and_one_comes_two(): + assert next_release(["sweet/2026.10.0", "sweet/2026.10.1"], "2026.10") == "sweet/2026.10.2" + + +def test_order_of_the_tag_list_does_not_matter(): + assert next_release(["sweet/2026.10.1", "sweet/2026.10.0"], "2026.10") == "sweet/2026.10.2" + + +def test_counting_is_numeric_past_nine(): + tags = [f"sweet/2026.10.{n}" for n in range(11)] + assert next_release(tags, "2026.10") == "sweet/2026.10.11" + + +def test_releases_from_other_months_are_ignored(): + tags = [ + "sweet/2026.09.0", + "sweet/2026.09.1", + "sweet/2026.09.2", + "sweet/2025.10.4", # same month, another year + "sweet/2026.11.0", + ] + assert next_release(tags, "2026.10") == "sweet/2026.10.0" + assert next_release(tags + ["sweet/2026.10.0"], "2026.10") == "sweet/2026.10.1" + + +@pytest.mark.parametrize( + "malformed", + [ + "sweet/2026.10.01", # leading zero: a second spelling of .1 + "sweet/2026.10", + "sweet/2026.10.", + "sweet/2026.10.x", + "sweet/2026.10.7-rc1", + "sweet/2026.1.7", + "sweet/26.10.7", + "sweet/test", + "SWEET/2026.10.7", + "release/2026.10.7", + "sweet/2026.10.7/extra", + "sweet/2026.10.٧", # a non-ASCII digit, which \d would have accepted + ], +) +def test_malformed_tags_are_ignored(malformed): + assert next_release(["sweet/2026.10.0", malformed], "2026.10") == "sweet/2026.10.1" + + +def test_a_gap_never_hands_out_a_used_number(): + """If .3 was named by hand after .0 and .1, the next is .4, not .2 or .3: a release + has to sort after every release before it.""" + tags = ["sweet/2026.10.0", "sweet/2026.10.1", "sweet/2026.10.3"] + assert next_release(tags, "2026.10") == "sweet/2026.10.4" + + +def test_surrounding_whitespace_in_the_tag_list_is_tolerated(): + assert next_release([" sweet/2026.10.0 ", ""], "2026.10") == "sweet/2026.10.1" + + +@pytest.mark.parametrize("month", ["2026.13", "2026.00", "2026-10", "2026.1", "", "latest"]) +def test_a_bad_month_is_refused(month): + with pytest.raises(ValueError): + next_release([], month) + + +@pytest.mark.parametrize( + "name, ok", + [ + ("sweet/2026.10.0", True), + ("sweet/2026.10.12", True), + ("sweet/2026.01.0", True), + ("sweet/2026.13.0", False), # matches the documented pattern; no such month + ("sweet/2026.00.0", False), + ("sweet/2026.10.01", False), + ("wastemap/2026.10.07", False), + ("2026.10.0", False), + ], +) +def test_parse_accepts_exactly_the_release_names(name, ok): + assert (next_release_mod.parse(name) is not None) is ok + + +def test_latest_is_numeric_and_ignores_everything_else(): + tags = OTHER_PROJECTS + [ + "sweet/2026.10.9", + "sweet/2026.10.10", + "sweet/2026.09.30", + "sweet/2025.12.99", + "sweet/test", + ] + assert latest_release(tags) == "sweet/2026.10.10" + + +def test_latest_of_no_releases_is_none(): + assert latest_release(OTHER_PROJECTS) is None + assert latest_release([]) is None + + +# --- the command line the workflow actually runs ----------------------------------- + + +def _run(*args, stdin=""): + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + input=stdin, + capture_output=True, + text=True, + check=False, + ) + + +def test_cli_next_reads_tags_from_stdin(): + out = _run("2026.10", stdin="sweet/2026.10.0\nwastemap/2026.10.07\nsweet/2026.10.1\n") + assert out.returncode == 0, out.stderr + assert out.stdout == "sweet/2026.10.2\n" + + +def test_cli_next_defaults_to_the_current_utc_month(): + before = next_release_mod._current_month() + out = _run(stdin="") + after = next_release_mod._current_month() # in case the month turned over mid-test + assert out.returncode == 0, out.stderr + assert out.stdout in {f"sweet/{before}.0\n", f"sweet/{after}.0\n"} + + +def test_cli_check_prints_the_trimmed_name_or_refuses(): + good = _run("--check", " sweet/2026.10.3 ") + assert (good.returncode, good.stdout) == (0, "sweet/2026.10.3\n") + + bad = _run("--check", "sweet/2026.10.03") + assert bad.returncode == 1 + assert bad.stdout == "" + assert "not a release name" in bad.stderr + + +def test_cli_latest_prints_nothing_when_there_is_no_release(): + assert _run("--latest", stdin="wastemap/2026.10.07\n").stdout == "" + assert _run("--latest", stdin="sweet/2026.09.0\nsweet/2026.10.0\n").stdout == "sweet/2026.10.0\n" + + +def test_cli_bad_month_exits_nonzero(): + out = _run("2026.13") + assert out.returncode == 2 + assert out.stdout == "" diff --git a/tests/test_site_type_defaults.py b/tests/test_site_type_defaults.py new file mode 100644 index 0000000..6ee1810 --- /dev/null +++ b/tests/test_site_type_defaults.py @@ -0,0 +1,163 @@ +"""The site type assumed for a site whose type nobody recorded. + +Before 2026-10-13 this guess was made by three different rules in nine places: +Climate TRACE's pipeline used income, region and named countries, while SWEET's +site and city paths used one of two region lists (one with Eastern Europe, one +with Australia and New Zealand instead). A Kazakh site with no type was a +sanitary landfill in Climate TRACE and an open dump in WasteMAP's site tool. +""" + +import inspect + +import numpy as np +import pandas as pd +import pytest + +import SWEET_python.city_params as city_params +import SWEET_python.site_type_defaults as std +from SWEET_python.country_income import INCOME_ID_BY_ISO3 + + +# --------------------------------------------------------------------------- # +# The rule +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize( + "iso3, why", + [ + ("AUS", "high income; its region is not in the list"), + ("ABW", "high income in the Caribbean"), + ("CHL", "high income in South America"), + ("POL", "Eastern Europe"), + ("KAZ", "Central Asia"), + ("TUR", "Western Asia"), + ("JPN", "Eastern Asia"), + ("MEX", "named: open-dumps 3% of its waste"), + ("BRA", "named"), + ("CRI", "named"), + ("CHN", "named, and Eastern Asia"), + ("RUS", "named, and Eastern Europe"), + ], +) +def test_engineered_disposal_systems_default_to_landfill(iso3, why): + assert std.default_site_type(iso3) == "Sanitary Landfill", why + + +@pytest.mark.parametrize("iso3", ["NGA", "IND", "IDN", "PAK", "EGY", "PER", "ZAF"]) +def test_other_countries_default_to_an_open_dump(iso3): + assert std.default_site_type(iso3) == "Dumpsite" + + +@pytest.mark.parametrize("iso3", ["MNG", "YEM"]) +def test_mongolia_and_yemen_are_dumps_whatever_their_region_says(iso3): + """Central Asia and Western Asia would otherwise sweep them in.""" + assert std.default_site_type(iso3) == "Dumpsite" + assert std.default_site_type(iso3, income_id="HIC") == "Dumpsite" + + +@pytest.mark.parametrize("iso3", [None, np.nan, pd.NA, "", "XXX"]) +def test_an_unknown_country_is_an_open_dump(iso3): + assert std.default_site_type(iso3) == "Dumpsite" + + +def test_a_lower_case_code_is_read_the_same(): + assert std.default_site_type("aus") == "Sanitary Landfill" + assert std.default_site_type("mng") == "Dumpsite" + + +def test_a_callers_own_income_is_used_when_it_has_one(): + """Climate TRACE passes the income on its site rows.""" + assert std.default_site_type("NGA", income_id="HIC") == "Sanitary Landfill" + assert std.default_site_type("ARG", income_id="UMC") == "Dumpsite" + + +@pytest.mark.parametrize("missing", [None, np.nan, pd.NA, ""]) +def test_a_missing_income_falls_back_to_sweets_table(missing): + assert std.default_site_type("AUS", income_id=missing) == "Sanitary Landfill" + assert std.default_site_type("NGA", income_id=missing) == "Dumpsite" + + +def test_landfill_is_default_agrees_with_default_site_type(): + for iso3 in INCOME_ID_BY_ISO3: + landfill = std.landfill_is_default(iso3) + assert std.default_site_type(iso3) == ("Sanitary Landfill" if landfill else "Dumpsite") + + +# --------------------------------------------------------------------------- # +# The income table +# --------------------------------------------------------------------------- # + +def test_the_income_table_is_the_one_climate_trace_reads(): + """217 countries, the What a Waste 2.0 classification (2018). + + Pinned on three countries the World Bank has reclassified since, so a swap to + the current list fails here instead of quietly disagreeing with Climate TRACE. + """ + assert len(INCOME_ID_BY_ISO3) == 217 + assert set(INCOME_ID_BY_ISO3.values()) == {"HIC", "UMC", "LMC", "LIC"} + assert INCOME_ID_BY_ISO3["ARG"] == "HIC" + assert INCOME_ID_BY_ISO3["BGR"] == "UMC" + assert INCOME_ID_BY_ISO3["CRI"] == "UMC" + + +def test_every_listed_country_has_a_sweet_region_except_the_channel_islands(): + from SWEET_python import defaults_2019 + + unplaced = set(INCOME_ID_BY_ISO3) - set(defaults_2019.region_lookup_iso3) + assert unplaced == {"CHI"} + + +# --------------------------------------------------------------------------- # +# Every caller uses it +# --------------------------------------------------------------------------- # + +@pytest.mark.parametrize( + "method, calls", + [ + ("load_andre_params", 3), # the city split's last fallback, three branches + ("site_only_estimate", 1), + ("site_only_estimate_trace", 1), + ("citysite_estimate_trace", 1), + ("sdst_prepopulate", 2), # by location, and a listed site with no type + ], +) +def test_each_path_that_guesses_a_type_calls_the_shared_rule(method, calls): + source = inspect.getsource(getattr(city_params.City, method)) + assert source.count("site_type_defaults.") == calls + assert "landfill_default_regions" not in source + assert '"Australia and New Zealand"' not in source + + +def test_a_site_with_no_type_is_typed_by_the_rule_end_to_end(): + """A Kazakh site with no type models as a sanitary landfill, MCF 1.0. + + Its region list used to make it an open dump (0.6) in this path while Climate + TRACE typed it a landfill. + """ + row = pd.Series( + { + "asset_identifier": 990001, + "iso3_country": "KAZ", + "type": np.nan, + "income_id": np.nan, + "annual_incoming_waste": 100_000.0, + "incoming_waste_year": 2023.0, + "site_open_year": 2000.0, + "site_close_year": 2050.0, + "waste_depth": np.nan, + "landfill_gas_collection": False, + "gas_collection_efficiency": 0.0, + "latitude": 51.0, + "longitude": 71.0, + "precipitation": 300.0, + "temperature": 5.0, + } + ) + pop_data = pd.DataFrame( + {"growth_rate_historic": [1.01], "growth_rate_future": [1.01]}, index=["KAZ"] + ) + city = city_params.City(city_name="990001") + city.site_only_estimate_trace(row, pop_data=pop_data) + landfill = city.baseline_parameters.landfills[0] + assert landfill.site_type == "Sanitary Landfill" + assert float(pd.Series(landfill.mcf).iloc[-1]) == 1.0 diff --git a/tests/test_version_stamp.py b/tests/test_version_stamp.py index f6f4770..0c73701 100644 --- a/tests/test_version_stamp.py +++ b/tests/test_version_stamp.py @@ -28,13 +28,13 @@ def _reimport(monkeypatch, **env): return importlib.reload(SWEET_python) -def test_an_env_sha_wins(monkeypatch): - """The image build knows the ref it actually installed and passes it in. That beats - reading a repository, because the image holds source, not a repository.""" - mod = _reimport(monkeypatch, SWEET_GIT_SHA="a" * 40) +def test_an_env_sha_wins_over_a_repository(monkeypatch): + """With no build stamp, the app's SWEET_GIT_SHA beats reading a repository, because + an image holds source, not a repository.""" + monkeypatch.setattr(SWEET_python, "_sha_from_build", lambda: None) + monkeypatch.setenv("SWEET_GIT_SHA", "a" * 40) - assert mod.__version__ == "a" * 40 - assert mod.VERSION_SOURCE == "env" + assert SWEET_python._resolve_version() == ("a" * 40, "env") def test_a_blank_env_sha_does_not_count(monkeypatch): @@ -43,7 +43,7 @@ def test_a_blank_env_sha_does_not_count(monkeypatch): mod = _reimport(monkeypatch, SWEET_GIT_SHA=" ") assert mod.__version__ != "" - assert mod.VERSION_SOURCE in {"git", "unavailable"} + assert mod.VERSION_SOURCE in {"build", "git", "unavailable"} def test_an_editable_checkout_reads_its_own_git(monkeypatch): @@ -62,7 +62,64 @@ def test_an_editable_checkout_reads_its_own_git(monkeypatch): assert mod.VERSION_SOURCE == "git" +def test_a_build_stamp_beats_the_environment(monkeypatch): + """A copy built from a git checkout carries its commit (setup.py writes + _build_sha.py). That describes the installed code; an env var is a claim from outside.""" + monkeypatch.setattr(SWEET_python, "_sha_from_build", lambda: "b" * 40) + monkeypatch.setenv("SWEET_GIT_SHA", "a" * 40) + + assert SWEET_python._resolve_version() == ("b" * 40, "build") + + +def test_without_a_stamp_the_environment_still_counts(monkeypatch): + monkeypatch.setattr(SWEET_python, "_sha_from_build", lambda: None) + monkeypatch.setenv("SWEET_GIT_SHA", "a" * 40) + + assert SWEET_python._resolve_version() == ("a" * 40, "env") + + +def test_the_source_tree_carries_no_stamp(): + """setup.py writes the stamp into the build directory only, so an editable install + of a checkout can never report a stale commit from one.""" + from pathlib import Path + + assert not (Path(SWEET_python.__file__).parent / "_build_sha.py").exists() + + +def test_setup_stamps_only_a_clean_checkout(tmp_path): + """The commit a build reports is the code it built: a dirty tree gets no stamp.""" + import importlib.util + from pathlib import Path + + spec = importlib.util.spec_from_file_location( + "sweet_setup", Path(SWEET_python.__file__).parents[1] / "setup.py" + ) + setup_src = spec.origin + namespace: dict = {} + # Only the helper, not setup() itself. + source = Path(setup_src).read_text() + helper = source[source.index("def _clean_head") : source.index("class build_py_with_commit")] + exec("import pathlib, subprocess\n" + helper, namespace) + clean_head = namespace["_clean_head"] + + def git(*args): + subprocess.run(["git", "-C", str(tmp_path), *args], check=True, capture_output=True) + + git("init", "-q") + git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "--allow-empty", "-m", "x") + (tmp_path / "f.txt").write_text("1") + git("add", "f.txt") + git("-c", "user.email=t@t", "-c", "user.name=t", "commit", "-q", "-m", "y") + head = subprocess.run(["git", "-C", str(tmp_path), "rev-parse", "HEAD"], capture_output=True, text=True).stdout.strip() + + assert clean_head(tmp_path) == head + (tmp_path / "f.txt").write_text("2") + assert clean_head(tmp_path) is None + assert clean_head(tmp_path / "not-a-repo") is None + + def test_an_unreadable_install_says_unknown_rather_than_guessing(monkeypatch): + monkeypatch.setattr(SWEET_python, "_sha_from_build", lambda: None) monkeypatch.setattr(SWEET_python, "_sha_from_git", lambda _: None) monkeypatch.delenv("SWEET_GIT_SHA", raising=False)