Check the HCAT tables as the converters apply them, online too

CHANGELOG.md

@@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [Unreleased]
 
 ### Added
+- Added `pixi run check-hcat`, which compares the HCAT mapping tables the converters read online with the taxonomy.
 - Added `FiboaDuckDBBaseConverter` for SQL-based conversion of large Parquet sources.
 - Added `PerFileBaseConverter` to process multi-file sources incrementally.
 - Added support for supplementary HCAT/crop mappings via `hcat_mapping_supplements`.

pyproject.toml

@@ -90,6 +90,8 @@ test = "pytest --cov=fiboa_cli --cov-report=term-missing --cov-report=html --cov
 lint = "ruff check --fix"
 format = "ruff format"
 check = {depends-on = ["lint", "format", "test"]}
+# Compare the HCAT mapping tables the converters read online with the taxonomy
+check-hcat = "python scripts/check_hcat.py"
 build = "python -m build"
 
 # Pre-commit tasks

scripts/check_hcat.py

@@ -0,0 +1,75 @@
+"""Check the HCAT mapping tables the converters read online against https://fiboa.org/code/hcat3.csv.
+
+The offline counterpart is tests/test_hcat_codes.py, which only sees the fixtures.
+Run with `pixi run check-hcat`; exits non-zero when a row disagrees with the taxonomy.
+"""
+
+import csv
+import io
+import sys
+from collections import defaultdict
+
+import requests
+
+from fiboa_cli.converters import Converters
+from fiboa_cli.datasets.commons.hcat import AddHCATMixin, hcat_mapping_url
+
+TAXONOMY = "https://fiboa.org/code/hcat3.csv"
+_tables = {}
+
+
+def fetch(url):
+    if url not in _tables:
+        response = requests.get(url, timeout=60)
+        if response.status_code != 200:
+            _tables[url] = None
+        else:
+            try:
+                text = response.content.decode("utf-8")
+            except UnicodeDecodeError:
+                text = response.content.decode("latin-1")
+            _tables[url] = list(csv.DictReader(io.StringIO(text)))
+    return _tables[url]
+
+
+def row_key(row):
+    return (row.get("original_code") or "").strip() or (row.get("original_name") or "").strip()
+
+
+def main():
+    taxonomy = {r["HCAT3_name"].strip(): r["HCAT3_code"].strip() for r in fetch(TAXONOMY)}
+    problems = defaultdict(set)
+    unreachable = defaultdict(set)
+    converters = Converters()
+    for _id in sorted(converters.list_ids()):
+        converter = converters.load(_id)
+        if not isinstance(converter, AddHCATMixin) or not converter.hcat_mapping_csv:
+            continue
+        rows = {}
+        for url in [converter.hcat_mapping_csv, *converter.hcat_mapping_supplements]:
+            table = fetch(hcat_mapping_url(url))
+            if table is None:
+                unreachable[hcat_mapping_url(url)].add(_id)
+                continue
+            rows |= {row_key(r): r for r in table}
+        for row in rows.values():
+            name = (row.get("HCAT3_name") or "").strip()
+            code = (row.get("HCAT3_code") or "").strip()
+            if not name and not code:
+                continue
+            if name not in taxonomy:
+                problems[f"{name!r} ({code}) is not an HCAT class"].add(_id)
+            elif taxonomy[name] != code:
+                problems[f"{name} has {code}, the taxonomy says {taxonomy[name]}"].add(_id)
+
+    for url, ids in sorted(unreachable.items()):
+        print(f"unreachable: {url} ({', '.join(sorted(ids))})")
+    for problem, ids in sorted(problems.items()):
+        print(f"{problem}: {', '.join(sorted(ids))}")
+    if not problems and not unreachable:
+        print("All mapping tables agree with the taxonomy.")
+    return 1 if problems or unreachable else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tests/data-files/convert/be_vlg/vlg_supplement.csv

@@ -0,0 +1,35 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+47,Goudsbloem,Marigold,calendula_marigold,3301061210
+48,Deder/huttentut,Camelina,camelina,3301061500
+391,Mengteelt andere wintergranen en winterteelt vlinderbloemige,Intercropping other winter cereals and winter legume,unspecified_cereals,3301011500
+392,Mengteelt zomergranen en zomerteelt vlinderbloemige,Intercropping spring cereals and spring legume,unspecified_cereals,3301011500
+395,Mengteelt wintertarwe of triticale en winterteelt vlinderbloemige,Intercropping winter wheat or triticale and winter legume,winter_common_soft_wheat,3301010101
+451,Wintervlas (olieproductie),Winter flax (oil production),flax_linseed_oil,3301060702
+511,Voedererwten (winterteelt) - niet menselijke consumptie,Fodder peas (winter) - not for human consumption,peas,3301020600
+512,Voedererwten (zomerteelt) - niet menselijke consumptie,Fodder peas (spring) - not for human consumption,peas,3301020600
+513,Drooggeoogste erwten (voor menselijke consumptie),Dry harvested peas (for human consumption),peas,3301020600
+521,Winterveldbonen - niet menselijke consumptie,Winter field beans - not for human consumption,beans,3301020100
+522,Zomerveldbonen - niet menselijke consumptie,Spring field beans - not for human consumption,beans,3301020100
+523,Drooggeoogste Vicia bonen (voor menselijke consumptie),Dry harvested Vicia beans (for human consumption),beans,3301020100
+524,Drooggeoogste Phaseolus bonen (voor menselijke consumptie),Dry harvested Phaseolus beans (for human consumption),beans,3301020100
+634,Mengsel met overwegend Japanse haver en vlinderbloemigen,Mixture predominantly Japanese oats with legumes,oats,3301010500
+635,Mengsel met overwegend Japanse haver zonder vlinderbloemigen,Mixture predominantly Japanese oats without legumes,oats,3301010500
+655,Sareptamosterd,Brown mustard,mustard,3301210100
+749,Mengsel met 1 of meerdere vlinderbloemigen,Mixture with one or more legumes,legumes_dried_pulses_protein_crops,3301020000
+858,Andere meerjarige kruiden - industrie,Other perennial herbs - industry,unspecified_aromatic_medicinal_culinary_plants_spices_herbs,3301061298
+881,Engelwortel - vers,Angelica - fresh,angelica,3301061204
+899,Rotatieteelt van groentegewassen (vb pluktuin),Rotational cultivation of vegetable crops,fresh_vegetables,3301070000
+907,Aardappelen (gecertificeerd pootgoed),Potatoes (certified seed),potatoes,3301030000
+908,Aardappelen (hoevepootgoed),Potatoes (farm-saved seed),potatoes,3301030000
+923,Wintervlas (vezelproductie en industriële toepassingen),Winter flax (fibre production and industrial use),flax_linen,3301060701
+933,Kikkererwten,Chickpeas,chickpeas,3301020200
+934,Linzen,Lentils,lentils,3301020500
+6041,Zaaizaad wintervlas,Seed of winter flax,flax_linen,3301060701
+8526,Chinese kool - industrie,Chinese cabbage - industry,chinese_cabbage,3301210205
+8531,Radijs - industrie,Radish - industry,radish,3301290600
+8532,Rode biet - industrie,Beetroot - industry,beetroot_beets,3301290200
+8545,Raketsla - Rucola - industrie,Rocket - industry,rocket_arugula,3301310600
+8552,Tomaten - industrie,Tomatoes - industry,tomato,3301280000
+8553,Paprika - industrie,Bell pepper - industry,bell_pepper_paprika,3301300100
+8581,Andere meerjarige kruiden - vers,Other perennial herbs - fresh,aromatic_medicinal_culinary_plants_spices_herbs,3301061200
+9571,Begonia voor de knol,Begonia for the tuber,begonias,3301080400

tests/data-files/convert/cz/cz_supplement.csv

@@ -167,3 +167,5 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code
 50371,Směs pro ozeleněný kolejový řádek,Mixture for a greened tramline,not_known_and_other,3399000000
 50372,Směs pro ochranný pás kolem KP,Protective strip mixture around a landscape feature,not_known_and_other,3399000000
 50374,Chřestnatec,Unidentified species (chřestnatec),not_known_and_other,3399000000
+389,Petržel kořenová,Petroselinum,parsly,3301061227
+390,Petržel naťová,Parsley,parsly,3301061227

tests/data-files/convert/de_bb/de_supplement.csv

@@ -0,0 +1,3 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+659,Petersilie,parsley/Petroselinum (parsley),parsly,3301061227
+860,Spargel,asparagus,asparagus,3301200000

tests/data-files/convert/dk/dk_supplement.csv

@@ -0,0 +1,32 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+19,Majs til modenhed med græsudlæg,Maize for maturity with grass undersowing,grain_maize_corn_popcorn,3301010600
+26,Linser,Lentils,lentils,3301020500
+41,Spindhør,Fibre flax,flax_linen,3301060701
+70,"Vårbyg, efterårssået","Spring barley, autumn sown",spring_barley,3301010402
+71,"Vårhvede, efterårssået","Spring wheat, autumn sown",spring_common_soft_wheat,3301010102
+72,"Vårhavre, efterårssået","Spring oats, autumn sown",spring_oats,3301010502
+127,Lucernefrø,Alfalfa seed,alfalfa_lucerne,3301090301
+154,"Kartofler, spise- (proces, skrællet kogte)","Potatoes, table (processed, peeled boiled)",potatoes,3301030000
+155,"Kartofler, pulver/granules-","Potatoes, powder/granules",potatoes,3301030000
+156,"Kartofler, friteret/chips/pommes frites","Potatoes, fried/chips",potatoes,3301030000
+157,"Kartofler, spise- tidligt høstede med efterafgrøder","Potatoes, table, early harvested with catch crops",potatoes,3301030000
+218,Majshelsæd med græsudlæg,Maize wholecrop with grass undersowing,green_silo_maize,3301090400
+224,"Blandkorn, efterårssået helsæd","Meslin, autumn sown wholecrop",winter_meslin,3301011101
+236,"Græs med kløver/lucerne, under 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, under 50% legume (rotation), autumn sown in winter cereal for green fodder",plants_harvested_green,3301090000
+237,"Græs med kløver/lucerne, over 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, over 50% legume (rotation), autumn sown in winter cereal for green fodder",clover,3301090303
+345,"Brak langs vandløb og søer, slåning (alternativ til efterafgrøder)","Fallow along watercourses and lakes, mown",fallow_land_not_crop,3301110000
+425,Sukkermajs med græsudlæg,Sweetcorn with grass undersowing,grain_maize_corn_popcorn,3301010600
+426,"Bønner, andre","Beans, other",beans,3301020100
+427,Babyleaves,Baby leaves,salads_lettuce_leaf_vegetables,3301310000
+428,Spidskål,Pointed cabbage,savoy_cabbage,3301210211
+482,Skovlandbrug med permanent græs,Agroforestry with permanent grass,tree_wood_forest,3306000000
+483,Skovlandbrug med græs i omdrift,Agroforestry with grass in rotation,tree_wood_forest,3306000000
+484,Skovlandbrug med omdriftsafgrøder,Agroforestry with rotational crops,tree_wood_forest,3306000000
+485,Skovlandbrug med permanente afgrøder,Agroforestry with permanent crops,tree_wood_forest,3306000000
+490,"Hassel, træ (Corylus avellana)","Hazel, tree (Corylus avellana)",hazelnuts_hazel,3303030200
+491,Storfrugtet tranebær,Large cranberry,cranberry,3303020500
+496,"Medicinpl., vedplanter","Medicinal plants, woody",aromatic_medicinal_culinary_plants_spices_herbs,3301061200
+535,Bærmispel,Juneberry,amelanchier_serviceberry,3303010100
+565,"Skovrejsning, direktivimplementerende uden tilsagn","Afforestation, directive-implementing without commitment",afforestation_reforestation,3306010000
+566,"Klimaskovrejsning, national ordning","Climate afforestation, national scheme",afforestation_reforestation,3306010000
+575,Skovrejsning (privat) - kulstofbinding og grundvandsbeskyttelse,Afforestation (private) - carbon capture and groundwater protection,afforestation_reforestation,3306010000

tests/data-files/convert/ec_si/si_2021_supplement.csv

@@ -0,0 +1,2 @@
+original_code,original_name,translated_name,HCAT3_name,HCAT3_code
+704,trsnice,Vine nurseries,nurseries_nursery,3303070000

tests/data-files/convert/ee/ee.csv

@@ -273,7 +273,7 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code,HCAT2_name,HCA
 0272,"sinerõigas, harilik",Common oil radish,radish,3301290600,radish,3301290600
 0273,"harilik sibul ""Jõgeva 3""","Onion ""Jõgeva 3""",onions,3301220400,onions,3301220400
 0274,"põldhernes ""Kirke"" tugikultuuriga (vähemalt 70% põldhernest ""Kirke"")","Field pea ""Kirke"" with support crop (at least 70%)",peas,3301020600,peas,3301020600
-0275,maguslupiin,Sweet lupin,lupins,3301020300,legumes_dried_pulses_protein_crops,3301020000
+0275,maguslupiin,Sweet lupin,sweet_lupins,3301020700,legumes_dried_pulses_protein_crops,3301020000
 0276,muu söödapeet,Other fodder beet,mangelwurzel_fodder_beet,3301290400,mangelwurzel_fodder_beet,3301290400
 0277,"suvitritikale allakülviga, v.a liblikõieliste allakülviga",Spring triticale undersown (excl. legumes),spring_triticale,3301010802,spring_triticale,3301010802
 0278,punane peakapsas,Red head cabbage,brassica_oleracea_cabbage,3301210200,brassica_oleracea_cabbage,3301210200
@@ -282,14 +282,14 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code,HCAT2_name,HCA
 0281,"mädarõigas, aed-",Garden horseradish,horseradish,3301210400,horseradish,3301210400
 0282,lehtpeet,Leaf beet (chard),beetroot_beets,3301290200,beetroot_beets,3301290200
 0283,"kartul ""Väike verev""","Potato ""Väike verev""",potatoes,3301030000,potatoes,3301030000
-0284,kiulina,Fibre flax,flax_fibre,3301060701,flax_fibre,3301060701
+0284,kiulina,Fibre flax,flax_linen,3301060701,flax_linen,3301060701
 0285,toompihlakas,Serviceberry (Amelanchier),berries_berry_species,3303020000,berries_berry_species,3303020000
 0286,"kanep, Bialobrzeskie",Hemp Bialobrzeskie,hemp_cannabis,3301061000,hemp_cannabis,3301061000
 0287,"nõges, kõrve-",Stinging nettle,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299,aromatic_medicinal_culinary_plants_spices_herbs,3301061200
 0288,"münt, vääris-",Apple mint,mints_peppermint,3301061222,mints_peppermint,3301061222
 0289,"monarda, sidrun-",Lemon beebalm,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299,aromatic_medicinal_culinary_plants_spices_herbs,3301061200
 0290,"münt, rohe-",Spearmint,mints_peppermint,3301061222,mints_peppermint,3301061222
-0291,sojauba,Soybean,soybeans,3301020700,legumes_dried_pulses_protein_crops,3301020000
+0291,sojauba,Soybean,soy_soybeans,3301160000,legumes_dried_pulses_protein_crops,3301020000
 0292,"harakputk, aed-",Garden chervil,chervil,3301061214,chervil,3301061214
 0293,harilik lutsern (100% lutsern),Alfalfa (100% alfalfa),alfalfa_lucerne,3301090301,alfalfa_lucerne,3301090301
 0294,"harilik nõiahammas (vähemalt 80% nõiahammast, kuni 20% heintaimi)",Birdsfoot trefoil (at least 80%),lotus,3301082600,lotus,3301082600

tests/data-files/convert/fr/fr.csv

@@ -306,7 +306,7 @@ TAB,Tabac,Tobacco,tobacco,3301060100,tobacco,3301060100,EuroCrops fr_2018.csv
 TCR,Taillis à courte rotation,Short rotation coppice,tree_wood_forest,3306000000,other_tree_wood_forest,3306990000,EuroCrops fr_2018.csv
 THY,Thym,Thyme,thyme,3301061237,thyme,3301061237,EuroCrops fr_2018.csv
 TOM,Tomate,Tomato,tomato,3301280000,tomato,3301280000,EuroCrops fr_2018.csv
-TOP,Topinambour,Jerusalem artichoke,topinambur_jerusalem_artichoke,3301290900,topinambur_jerusalem_artichoke,3301290900,EuroCrops fr_2018.csv
+TOP,Topinambour,Jerusalem artichoke,topinambur_jerusalem_artichoke,3301180000,topinambur_jerusalem_artichoke,3301180000,EuroCrops fr_2018.csv
 TOT,Tomate pour transformation,Tomato for processing,tomato,3301280000,tomato,3301280000,EuroCrops fr_2018.csv
 TR5,Trèfle implante pour la recolte 2015,Clover planted for the 2015 harvest,clover,3301090303,clover,3301090303,EuroCrops fr_other_years.csv
 TR6,Trèfle implanté pour la récolte 2016,Clover planted for the 2016 harvest,clover,3301090303,clover,3301090303,EuroCrops fr_2018.csv