diff --git a/CHANGELOG.md b/CHANGELOG.md index c02ea68d..66f09e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ## [Unreleased] ### Added +- Added `pixi run check-hcat`, which compares the HCAT mapping tables the converters read online with the taxonomy. - Added `FiboaDuckDBBaseConverter` for SQL-based conversion of large Parquet sources. - Added `PerFileBaseConverter` to process multi-file sources incrementally. - Added support for supplementary HCAT/crop mappings via `hcat_mapping_supplements`. diff --git a/pixi.lock b/pixi.lock index d36dc2e9..92f7a40d 100644 --- a/pixi.lock +++ b/pixi.lock @@ -4765,7 +4765,7 @@ packages: - pypi: ./ name: fiboa-cli version: 0.21.0 - sha256: 537013851e61795948896221dda7462beebd711be58e95b5eab43c9542e9f3e0 + sha256: 6a751743588959508c3a958910a711b1c578818b4b9226a7b60b2e2de64d52ad requires_dist: - vecorel-cli==0.3.0 - beautifulsoup4>=4.12 diff --git a/pyproject.toml b/pyproject.toml index 98f2634e..5fc90aa8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,6 +90,8 @@ test = "pytest --cov=fiboa_cli --cov-report=term-missing --cov-report=html --cov lint = "ruff check --fix" format = "ruff format" check = {depends-on = ["lint", "format", "test"]} +# Compare the HCAT mapping tables the converters read online with the taxonomy +check-hcat = "python scripts/check_hcat.py" build = "python -m build" # Pre-commit tasks diff --git a/scripts/check_hcat.py b/scripts/check_hcat.py new file mode 100644 index 00000000..92595050 --- /dev/null +++ b/scripts/check_hcat.py @@ -0,0 +1,75 @@ +"""Check the HCAT mapping tables the converters read online against https://fiboa.org/code/hcat3.csv. + +The offline counterpart is tests/test_hcat_codes.py, which only sees the fixtures. +Run with `pixi run check-hcat`; exits non-zero when a row disagrees with the taxonomy. +""" + +import csv +import io +import sys +from collections import defaultdict + +import requests + +from fiboa_cli.converters import Converters +from fiboa_cli.datasets.commons.hcat import AddHCATMixin, hcat_mapping_url + +TAXONOMY = "https://fiboa.org/code/hcat3.csv" +_tables = {} + + +def fetch(url): + if url not in _tables: + response = requests.get(url, timeout=60) + if response.status_code != 200: + _tables[url] = None + else: + try: + text = response.content.decode("utf-8") + except UnicodeDecodeError: + text = response.content.decode("latin-1") + _tables[url] = list(csv.DictReader(io.StringIO(text))) + return _tables[url] + + +def row_key(row): + return (row.get("original_code") or "").strip() or (row.get("original_name") or "").strip() + + +def main(): + taxonomy = {r["HCAT3_name"].strip(): r["HCAT3_code"].strip() for r in fetch(TAXONOMY)} + problems = defaultdict(set) + unreachable = defaultdict(set) + converters = Converters() + for _id in sorted(converters.list_ids()): + converter = converters.load(_id) + if not isinstance(converter, AddHCATMixin) or not converter.hcat_mapping_csv: + continue + rows = {} + for url in [converter.hcat_mapping_csv, *converter.hcat_mapping_supplements]: + table = fetch(hcat_mapping_url(url)) + if table is None: + unreachable[hcat_mapping_url(url)].add(_id) + continue + rows |= {row_key(r): r for r in table} + for row in rows.values(): + name = (row.get("HCAT3_name") or "").strip() + code = (row.get("HCAT3_code") or "").strip() + if not name and not code: + continue + if name not in taxonomy: + problems[f"{name!r} ({code}) is not an HCAT class"].add(_id) + elif taxonomy[name] != code: + problems[f"{name} has {code}, the taxonomy says {taxonomy[name]}"].add(_id) + + for url, ids in sorted(unreachable.items()): + print(f"unreachable: {url} ({', '.join(sorted(ids))})") + for problem, ids in sorted(problems.items()): + print(f"{problem}: {', '.join(sorted(ids))}") + if not problems and not unreachable: + print("All mapping tables agree with the taxonomy.") + return 1 if problems or unreachable else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/data-files/convert/be_vlg/vlg_supplement.csv b/tests/data-files/convert/be_vlg/vlg_supplement.csv new file mode 100644 index 00000000..023028a3 --- /dev/null +++ b/tests/data-files/convert/be_vlg/vlg_supplement.csv @@ -0,0 +1,35 @@ +original_code,original_name,translated_name,HCAT3_name,HCAT3_code +47,Goudsbloem,Marigold,calendula_marigold,3301061210 +48,Deder/huttentut,Camelina,camelina,3301061500 +391,Mengteelt andere wintergranen en winterteelt vlinderbloemige,Intercropping other winter cereals and winter legume,unspecified_cereals,3301011500 +392,Mengteelt zomergranen en zomerteelt vlinderbloemige,Intercropping spring cereals and spring legume,unspecified_cereals,3301011500 +395,Mengteelt wintertarwe of triticale en winterteelt vlinderbloemige,Intercropping winter wheat or triticale and winter legume,winter_common_soft_wheat,3301010101 +451,Wintervlas (olieproductie),Winter flax (oil production),flax_linseed_oil,3301060702 +511,Voedererwten (winterteelt) - niet menselijke consumptie,Fodder peas (winter) - not for human consumption,peas,3301020600 +512,Voedererwten (zomerteelt) - niet menselijke consumptie,Fodder peas (spring) - not for human consumption,peas,3301020600 +513,Drooggeoogste erwten (voor menselijke consumptie),Dry harvested peas (for human consumption),peas,3301020600 +521,Winterveldbonen - niet menselijke consumptie,Winter field beans - not for human consumption,beans,3301020100 +522,Zomerveldbonen - niet menselijke consumptie,Spring field beans - not for human consumption,beans,3301020100 +523,Drooggeoogste Vicia bonen (voor menselijke consumptie),Dry harvested Vicia beans (for human consumption),beans,3301020100 +524,Drooggeoogste Phaseolus bonen (voor menselijke consumptie),Dry harvested Phaseolus beans (for human consumption),beans,3301020100 +634,Mengsel met overwegend Japanse haver en vlinderbloemigen,Mixture predominantly Japanese oats with legumes,oats,3301010500 +635,Mengsel met overwegend Japanse haver zonder vlinderbloemigen,Mixture predominantly Japanese oats without legumes,oats,3301010500 +655,Sareptamosterd,Brown mustard,mustard,3301210100 +749,Mengsel met 1 of meerdere vlinderbloemigen,Mixture with one or more legumes,legumes_dried_pulses_protein_crops,3301020000 +858,Andere meerjarige kruiden - industrie,Other perennial herbs - industry,unspecified_aromatic_medicinal_culinary_plants_spices_herbs,3301061298 +881,Engelwortel - vers,Angelica - fresh,angelica,3301061204 +899,Rotatieteelt van groentegewassen (vb pluktuin),Rotational cultivation of vegetable crops,fresh_vegetables,3301070000 +907,Aardappelen (gecertificeerd pootgoed),Potatoes (certified seed),potatoes,3301030000 +908,Aardappelen (hoevepootgoed),Potatoes (farm-saved seed),potatoes,3301030000 +923,Wintervlas (vezelproductie en industriële toepassingen),Winter flax (fibre production and industrial use),flax_linen,3301060701 +933,Kikkererwten,Chickpeas,chickpeas,3301020200 +934,Linzen,Lentils,lentils,3301020500 +6041,Zaaizaad wintervlas,Seed of winter flax,flax_linen,3301060701 +8526,Chinese kool - industrie,Chinese cabbage - industry,chinese_cabbage,3301210205 +8531,Radijs - industrie,Radish - industry,radish,3301290600 +8532,Rode biet - industrie,Beetroot - industry,beetroot_beets,3301290200 +8545,Raketsla - Rucola - industrie,Rocket - industry,rocket_arugula,3301310600 +8552,Tomaten - industrie,Tomatoes - industry,tomato,3301280000 +8553,Paprika - industrie,Bell pepper - industry,bell_pepper_paprika,3301300100 +8581,Andere meerjarige kruiden - vers,Other perennial herbs - fresh,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 +9571,Begonia voor de knol,Begonia for the tuber,begonias,3301080400 diff --git a/tests/data-files/convert/cz/cz_supplement.csv b/tests/data-files/convert/cz/cz_supplement.csv index 447b2b76..377c2b4d 100644 --- a/tests/data-files/convert/cz/cz_supplement.csv +++ b/tests/data-files/convert/cz/cz_supplement.csv @@ -167,3 +167,5 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 50371,Směs pro ozeleněný kolejový řádek,Mixture for a greened tramline,not_known_and_other,3399000000 50372,Směs pro ochranný pás kolem KP,Protective strip mixture around a landscape feature,not_known_and_other,3399000000 50374,Chřestnatec,Unidentified species (chřestnatec),not_known_and_other,3399000000 +389,Petržel kořenová,Petroselinum,parsly,3301061227 +390,Petržel naťová,Parsley,parsly,3301061227 diff --git a/tests/data-files/convert/de_bb/de_supplement.csv b/tests/data-files/convert/de_bb/de_supplement.csv new file mode 100644 index 00000000..cbba0d54 --- /dev/null +++ b/tests/data-files/convert/de_bb/de_supplement.csv @@ -0,0 +1,3 @@ +original_code,original_name,translated_name,HCAT3_name,HCAT3_code +659,Petersilie,parsley/Petroselinum (parsley),parsly,3301061227 +860,Spargel,asparagus,asparagus,3301200000 diff --git a/tests/data-files/convert/dk/dk_supplement.csv b/tests/data-files/convert/dk/dk_supplement.csv new file mode 100644 index 00000000..ad17ed4f --- /dev/null +++ b/tests/data-files/convert/dk/dk_supplement.csv @@ -0,0 +1,32 @@ +original_code,original_name,translated_name,HCAT3_name,HCAT3_code +19,Majs til modenhed med græsudlæg,Maize for maturity with grass undersowing,grain_maize_corn_popcorn,3301010600 +26,Linser,Lentils,lentils,3301020500 +41,Spindhør,Fibre flax,flax_linen,3301060701 +70,"Vårbyg, efterårssået","Spring barley, autumn sown",spring_barley,3301010402 +71,"Vårhvede, efterårssået","Spring wheat, autumn sown",spring_common_soft_wheat,3301010102 +72,"Vårhavre, efterårssået","Spring oats, autumn sown",spring_oats,3301010502 +127,Lucernefrø,Alfalfa seed,alfalfa_lucerne,3301090301 +154,"Kartofler, spise- (proces, skrællet kogte)","Potatoes, table (processed, peeled boiled)",potatoes,3301030000 +155,"Kartofler, pulver/granules-","Potatoes, powder/granules",potatoes,3301030000 +156,"Kartofler, friteret/chips/pommes frites","Potatoes, fried/chips",potatoes,3301030000 +157,"Kartofler, spise- tidligt høstede med efterafgrøder","Potatoes, table, early harvested with catch crops",potatoes,3301030000 +218,Majshelsæd med græsudlæg,Maize wholecrop with grass undersowing,green_silo_maize,3301090400 +224,"Blandkorn, efterårssået helsæd","Meslin, autumn sown wholecrop",winter_meslin,3301011101 +236,"Græs med kløver/lucerne, under 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, under 50% legume (rotation), autumn sown in winter cereal for green fodder",plants_harvested_green,3301090000 +237,"Græs med kløver/lucerne, over 50 % bælgpl. (omdrift) efterårsudlagt i vinterkorn til grønkorn","Grass with clover/alfalfa, over 50% legume (rotation), autumn sown in winter cereal for green fodder",clover,3301090303 +345,"Brak langs vandløb og søer, slåning (alternativ til efterafgrøder)","Fallow along watercourses and lakes, mown",fallow_land_not_crop,3301110000 +425,Sukkermajs med græsudlæg,Sweetcorn with grass undersowing,grain_maize_corn_popcorn,3301010600 +426,"Bønner, andre","Beans, other",beans,3301020100 +427,Babyleaves,Baby leaves,salads_lettuce_leaf_vegetables,3301310000 +428,Spidskål,Pointed cabbage,savoy_cabbage,3301210211 +482,Skovlandbrug med permanent græs,Agroforestry with permanent grass,tree_wood_forest,3306000000 +483,Skovlandbrug med græs i omdrift,Agroforestry with grass in rotation,tree_wood_forest,3306000000 +484,Skovlandbrug med omdriftsafgrøder,Agroforestry with rotational crops,tree_wood_forest,3306000000 +485,Skovlandbrug med permanente afgrøder,Agroforestry with permanent crops,tree_wood_forest,3306000000 +490,"Hassel, træ (Corylus avellana)","Hazel, tree (Corylus avellana)",hazelnuts_hazel,3303030200 +491,Storfrugtet tranebær,Large cranberry,cranberry,3303020500 +496,"Medicinpl., vedplanter","Medicinal plants, woody",aromatic_medicinal_culinary_plants_spices_herbs,3301061200 +535,Bærmispel,Juneberry,amelanchier_serviceberry,3303010100 +565,"Skovrejsning, direktivimplementerende uden tilsagn","Afforestation, directive-implementing without commitment",afforestation_reforestation,3306010000 +566,"Klimaskovrejsning, national ordning","Climate afforestation, national scheme",afforestation_reforestation,3306010000 +575,Skovrejsning (privat) - kulstofbinding og grundvandsbeskyttelse,Afforestation (private) - carbon capture and groundwater protection,afforestation_reforestation,3306010000 diff --git a/tests/data-files/convert/ec_si/si_2021_supplement.csv b/tests/data-files/convert/ec_si/si_2021_supplement.csv new file mode 100644 index 00000000..d36b2893 --- /dev/null +++ b/tests/data-files/convert/ec_si/si_2021_supplement.csv @@ -0,0 +1,2 @@ +original_code,original_name,translated_name,HCAT3_name,HCAT3_code +704,trsnice,Vine nurseries,nurseries_nursery,3303070000 diff --git a/tests/data-files/convert/ee/ee.csv b/tests/data-files/convert/ee/ee.csv index 35f7fb11..95c3483e 100644 --- a/tests/data-files/convert/ee/ee.csv +++ b/tests/data-files/convert/ee/ee.csv @@ -273,7 +273,7 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code,HCAT2_name,HCA 0272,"sinerõigas, harilik",Common oil radish,radish,3301290600,radish,3301290600 0273,"harilik sibul ""Jõgeva 3""","Onion ""Jõgeva 3""",onions,3301220400,onions,3301220400 0274,"põldhernes ""Kirke"" tugikultuuriga (vähemalt 70% põldhernest ""Kirke"")","Field pea ""Kirke"" with support crop (at least 70%)",peas,3301020600,peas,3301020600 -0275,maguslupiin,Sweet lupin,lupins,3301020300,legumes_dried_pulses_protein_crops,3301020000 +0275,maguslupiin,Sweet lupin,sweet_lupins,3301020700,legumes_dried_pulses_protein_crops,3301020000 0276,muu söödapeet,Other fodder beet,mangelwurzel_fodder_beet,3301290400,mangelwurzel_fodder_beet,3301290400 0277,"suvitritikale allakülviga, v.a liblikõieliste allakülviga",Spring triticale undersown (excl. legumes),spring_triticale,3301010802,spring_triticale,3301010802 0278,punane peakapsas,Red head cabbage,brassica_oleracea_cabbage,3301210200,brassica_oleracea_cabbage,3301210200 @@ -282,14 +282,14 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code,HCAT2_name,HCA 0281,"mädarõigas, aed-",Garden horseradish,horseradish,3301210400,horseradish,3301210400 0282,lehtpeet,Leaf beet (chard),beetroot_beets,3301290200,beetroot_beets,3301290200 0283,"kartul ""Väike verev""","Potato ""Väike verev""",potatoes,3301030000,potatoes,3301030000 -0284,kiulina,Fibre flax,flax_fibre,3301060701,flax_fibre,3301060701 +0284,kiulina,Fibre flax,flax_linen,3301060701,flax_linen,3301060701 0285,toompihlakas,Serviceberry (Amelanchier),berries_berry_species,3303020000,berries_berry_species,3303020000 0286,"kanep, Bialobrzeskie",Hemp Bialobrzeskie,hemp_cannabis,3301061000,hemp_cannabis,3301061000 0287,"nõges, kõrve-",Stinging nettle,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 0288,"münt, vääris-",Apple mint,mints_peppermint,3301061222,mints_peppermint,3301061222 0289,"monarda, sidrun-",Lemon beebalm,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 0290,"münt, rohe-",Spearmint,mints_peppermint,3301061222,mints_peppermint,3301061222 -0291,sojauba,Soybean,soybeans,3301020700,legumes_dried_pulses_protein_crops,3301020000 +0291,sojauba,Soybean,soy_soybeans,3301160000,legumes_dried_pulses_protein_crops,3301020000 0292,"harakputk, aed-",Garden chervil,chervil,3301061214,chervil,3301061214 0293,harilik lutsern (100% lutsern),Alfalfa (100% alfalfa),alfalfa_lucerne,3301090301,alfalfa_lucerne,3301090301 0294,"harilik nõiahammas (vähemalt 80% nõiahammast, kuni 20% heintaimi)",Birdsfoot trefoil (at least 80%),lotus,3301082600,lotus,3301082600 diff --git a/tests/data-files/convert/fr/fr.csv b/tests/data-files/convert/fr/fr.csv index 0f5e8a69..201ccfda 100644 --- a/tests/data-files/convert/fr/fr.csv +++ b/tests/data-files/convert/fr/fr.csv @@ -306,7 +306,7 @@ TAB,Tabac,Tobacco,tobacco,3301060100,tobacco,3301060100,EuroCrops fr_2018.csv TCR,Taillis à courte rotation,Short rotation coppice,tree_wood_forest,3306000000,other_tree_wood_forest,3306990000,EuroCrops fr_2018.csv THY,Thym,Thyme,thyme,3301061237,thyme,3301061237,EuroCrops fr_2018.csv TOM,Tomate,Tomato,tomato,3301280000,tomato,3301280000,EuroCrops fr_2018.csv -TOP,Topinambour,Jerusalem artichoke,topinambur_jerusalem_artichoke,3301290900,topinambur_jerusalem_artichoke,3301290900,EuroCrops fr_2018.csv +TOP,Topinambour,Jerusalem artichoke,topinambur_jerusalem_artichoke,3301180000,topinambur_jerusalem_artichoke,3301180000,EuroCrops fr_2018.csv TOT,Tomate pour transformation,Tomato for processing,tomato,3301280000,tomato,3301280000,EuroCrops fr_2018.csv TR5,Trèfle implante pour la recolte 2015,Clover planted for the 2015 harvest,clover,3301090303,clover,3301090303,EuroCrops fr_other_years.csv TR6,Trèfle implanté pour la récolte 2016,Clover planted for the 2016 harvest,clover,3301090303,clover,3301090303,EuroCrops fr_2018.csv diff --git a/tests/data-files/convert/nl/nl.csv b/tests/data-files/convert/nl/nl.csv index 93dc00e7..7e36f2ba 100644 --- a/tests/data-files/convert/nl/nl.csv +++ b/tests/data-files/convert/nl/nl.csv @@ -83,8 +83,8 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 968,"Gladiool, droogbloemen",Gladiolus dried flowers,gladiolus_gladioli,3301082100 970,"Hyacint, overige bloemkwekerijgewassen",Hyacinth other flower nursery crops,other_flowers_ornamental_plants,3301089900 971,"Hyacint, droogbloemen",Hyacinth dried flowers,other_flowers_ornamental_plants,3301089900 -973,"Iris, overige bloemkwekerijgewassen",Iris other flower nursery crops,iris,3301082200 -974,"Iris, droogbloemen",Iris dried flowers,iris,3301082200 +973,"Iris, overige bloemkwekerijgewassen",Iris other flower nursery crops,iris,3301082300 +974,"Iris, droogbloemen",Iris dried flowers,iris,3301082300 976,"Krokus, overige bloemkwekerijgewassen",Crocus other flower nursery crops,saffron_crocus_sativus,3301061232 979,"Lelie, overige bloemkwekerijgewassen",Lily other flower nursery crops,lilies,3301082500 982,"Narcis, overige bloemkwekerijgewassen",Narcissus other flower nursery crops,narcissus_daffodil,3301083300 @@ -123,8 +123,8 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 1033,Igniscum Candy,Igniscum Candy,igniscum_candy,3304020100 1034,Kanariezaad,canary seed,canary_seed_canaryseed,3301011400 1035,Naaldaar (Setaria),Needle (Setaria),setaria,3301090206 -1036,Wortelpeterselie,parsley root,parsley,3301061227 -1037,"Peterselie, productie",Parsley production,parsley,3301061227 +1036,Wortelpeterselie,parsley root,parsly,3301061227 +1037,"Peterselie, productie",Parsley production,parsly,3301061227 1039,"Chrysant, overige bloemkwekerijgewassen",Chrysanthemum other flower nursery crops,chrysanthemum,3301080900 1042,"Angelica, productie",Angelica production,angelica,3301061204 1044,Papaver,Poppy,poppy,3301060600 @@ -133,8 +133,8 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 1047,Cranberry,cranberry,cranberry,3303020500 1048,"Echinacea (zonnehoed), productie",Echinacea (coneflower) production,echinacea_sun_hat,3301081500 1050,Leeuwenbekjes,Snapdragons,snapdragons,3301084500 -1051,"Iris, Bolvormend",Iris Spherical,iris,3301082200 -1052,"Iris, Rhizoomvormend",Iris Rhizomatous,iris,3301082200 +1051,"Iris, Bolvormend",Iris Spherical,iris,3301082300 +1052,"Iris, Rhizoomvormend",Iris Rhizomatous,iris,3301082300 1053,"Palmen, pot- en containervelden",Palms pot and container fields,other_tree_wood_forest,3306990000 1054,"Pioenroos, vermeerdering",Peony propagation,peony_peonies,3301083400 1067,"Bos- en haagplanten, open grond,",Forest and hedge plants open ground,tree_wood_forest,3306000000 @@ -188,7 +188,7 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 1935,Maiskolvesilage,Corn silage,green_silo_maize,3301090400 1936,"Bos, blijvend, met herplantplicht",Forest permanent with replanting obligation,tree_wood_forest,3306000000 1940,Voedselbos,food forest,tree_wood_forest,3306000000 -1949,Aardperen,Topinambur,topinambur_jerusalem_artichoke,3301290900 +1949,Aardperen,Topinambur,topinambur_jerusalem_artichoke,3301180000 2014,"Aardappelen, consumptie",Potatoes consumption,potatoes,3301030000 2015,"Aardappelen, poot NAK",Potatoes leg NAK,potatoes,3301030000 2016,"Aardappelen, poot TBM",Potatoes leg TBM,potatoes,3301030000 @@ -397,11 +397,11 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 6803,"Sneeuwklokje, bloembollen en -knollen",Snowdrop flower bulbs and tubers,other_flowers_ornamental_plants,3301089900 6746,"Beemdlangbloem, graszaad",Meadow foxtail grass seed,pasture_meadow_grassland_grass,3302000000 6798,Perceel bedekt met gewasresten (mulching),Plot covered with crop residues (mulching),fallow_land_not_crop,3301110000 -1038,"Peterselie, zaden en opkweekmateriaal",Parsley seeds and propagation material,parsley,3301061227 +1038,"Peterselie, zaden en opkweekmateriaal",Parsley seeds and propagation material,parsly,3301061227 6757,"Klaver, Alexandrijnse, klaverzaad",Alexandrian clover seed,clover,3301090303 983,"Narcis, droogbloemen",Daffodil dried flowers,narcissus_daffodil,3301083300 6786,"Timothee, graszaad",Timothy grass seed,timothy,3301090209 -6749,Bieslook,Chives,parsley,3301061227 +6749,Bieslook,Chives,chives,3301220100 6753,"Festulolium, groenbemesting, vanggewas",Festulolium green manure catch crop,festulolium,3301090204 2730,"Komkommer, zaden en opkweekmateriaal",Cucumber seeds and propagation material,cucumber_pickle,3301140100 2733,"Meloen, productie",Melon production,fresh_vegetables,3301070000 @@ -413,3 +413,14 @@ original_code,original_name,translated_name,HCAT3_name,HCAT3_code 986,"Tulp, droogbloemen",Tulip dried flowers,tulips,3301084900 2740,"Koolrabi, zaden en opkweekmateriaal",Kohlrabi seeds and propagation material,kohlrabi,3301210209 6761,"Klaver, Perzische, klaverzaad",Persian clover seed,clover,3301090303 +669,Zwaardherik (aaltjesvanggewas),Rocket (nematode catch crop),rocket_arugula,3301310600 +1011,"Sierui, droogbloemen",Ornamental onions dried flowers,other_flowers_ornamental_plants,3301089900 +1055,"Chrysant, vermeerdering",Chrysanthemum propagation,chrysanthemum,3301080900 +2738,"Koolraap, zaden en opkweekmateriaal",Rutabaga seeds and propagation material,swede_rutabaga,3301210500 +7136,Anna paulowna,Paulownia,other_tree_wood_forest,3306990000 +7139,"Notenbomen, hazelnoot",Nut trees hazelnut,hazelnuts_hazel,3303030200 +7193,"Notenbomen, walnoot",Nut trees walnut,walnuts,3303030600 +7194,"Notenbomen, overig",Nut trees other,nuts,3303030000 +7195,Bosui,Bunching onion,scallion,3301220500 +7196,Lente-ui,Spring onion,scallion,3301220500 +7197,Bamboe,Bamboo,poaceae_grasses,3301090200 diff --git a/tests/data-files/convert/si/si.csv b/tests/data-files/convert/si/si.csv index ca026ab0..eb630158 100644 --- a/tests/data-files/convert/si/si.csv +++ b/tests/data-files/convert/si/si.csv @@ -124,7 +124,7 @@ original_code,original_name,latin_name,translated_name,HCAT3_name,HCAT3_code 699,mešane sadne vrste,Mixed fruit plants,Mixed fruit plants,orchards_fruits,3303010000 702,drevesnice,Nurseries,Nurseries,nurseries_nursery,3303070000 703,šparglji,Asparagus officinalis,Garden asparagus,asparagus,3301200000 -704,trsnice,Vine nurseries,Vine nurseries,vineyards_wine_vine_rebland_grapes,3303070000 +704,trsnice,Vine nurseries,Vine nurseries,nurseries_nursery,3303070000 705,"mešane trajne rastline pod 0,1 ha","Mixed permanent crops on field under 0,1 ha size","Mixed permanent crops on field under 0,1 ha size",permanent_crops_perennial,3303000000 706,"trta za drugo rabo, ki ni vino ali namizno grozdje",Vines for other uses than for wine or fruits,Vines for other uses than for wine or fruits,vineyards_wine_vine_rebland_grapes,3303060000 707,matičnjak,Root-stock nursery,Root-stock nursery,nurseries_nursery,3303070000 @@ -171,10 +171,10 @@ original_code,original_name,latin_name,translated_name,HCAT3_name,HCAT3_code 399,"čičerika od 0,1 ha","Cicer arietinum (from 0.1 ha size)",Chickpeas (from 0.1 ha),chickpeas,3301020200 723,tobakovec,Nicotiana tabacum L.,Tobacco,other_arable_land_crops,3301990000 810,grašica (ozimna),Vicia villosa R.,Winter vetch,peas,3301020600 -635,pistacija,Pistacia vera,Pistachio,pistachios,3303030400 +635,pistacija,Pistacia vera,Pistachio,pistachio,3303030400 023,kvinoja,Chenopodium quinoa,Quinoa,other_cereals,3301019900 395,"bob od 0,1 ha","Vicia faba (from 0,1 ha size)",Field bean (from 0.1 ha),beans,3301020100 220,aleksandrijska detelja,Trifolium alexandrinum L,Egyptian clover,clover,3301090303 396,"grah od 0,1 ha","Pisum sativum (from 0,1 ha size)",Pea (from 0.1 ha),peas,3301020600 -398,"leča od 0,1 ha","Lens culinaris (from 0,1 ha size)",Lentils (from 0.1 ha),lentils,3301020300 +398,"leča od 0,1 ha","Lens culinaris (from 0,1 ha size)",Lentils (from 0.1 ha),lentils,3301020500 021,soržica (jara),Meslin,Meslin (spring),spring_meslin,3301011102 diff --git a/tests/data-files/convert/sk/sk.csv b/tests/data-files/convert/sk/sk.csv index 8f0eff8c..f5b0e4ea 100644 --- a/tests/data-files/convert/sk/sk.csv +++ b/tests/data-files/convert/sk/sk.csv @@ -76,7 +76,7 @@ Lupina biela,White lupine,sweet_lupins,3301020700,sweet_lupins,3301020700 Repa obyčajná cviklová (Cvikla),Beetroot (Beetroot),beetroot_beets,3301290200,beetroot_beets,3301290200 Ďatelina hybridná,Hybrid clover,clover,3301090303,clover,3301090303 Rajčiak jedlý,Edible tomato,tomato,3301280000,tomato,3301280000 -Petržlen záhradný,Garden parsley,parsley,3301061227,parsley,3301061227 +Petržlen záhradný,Garden parsley,parsly,3301061227,parsly,3301061227 Čerešňa vtáčia,Bird cherry,cherry_cherries,3303010400,cherry_cherries,3303010400 Cibuľa (zimná),Scallion,scallion,3301220500,onions,3301220400 Repa kŕmna,Fodder beet,mangelwurzel_fodder_beet,3301290400,mangelwurzel_fodder_beet,3301290400 @@ -202,7 +202,7 @@ Vinohrad v reštrukturalizácii,Vineyard under restructuring,vineyards_wine_vine Ostatná manipulačná plocha,Other handling area,not_known_and_other,3399000000,not_known_and_other,3399000000 Strukovinovo-obilná miešanka,Legume-cereal mixture,other_arable_land_crops,3301990000,other_arable_land_crops,3301990000 Olejninovo-obilná miešanka,Oilseed-cereal mixture,other_arable_land_crops,3301990000,other_arable_land_crops,3301990000 -Cuketa,Zucchini,zucchini_courgette,3301140200,zucchini_courgette,3301140200 +Cuketa,Zucchini,zucchini_courgette,3301140600,zucchini_courgette,3301140600 Cirok s podsevom,Sorghum with undersowing,millet_sorghum,3301010900,millet_sorghum,3301010900 Pamajorán obyčajný (oregano),Oregano,aromatic_medicinal_culinary_plants_spices_herbs,3301061200,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 Šalvia muškátová,Clary sage,aromatic_medicinal_culinary_plants_spices_herbs,3301061200,aromatic_medicinal_culinary_plants_spices_herbs,3301061200 @@ -242,4 +242,79 @@ Echinacea purpurová,Purple coneflower,echinacea_sun_hat,3301081500,echinacea_su Čakanka obyčajná šalátová,Chicory (salad),chicory_chicories,3301310200,chicory_chicories,3301310200 Kapusta repková kvaková (kvaka),Swede (rutabaga),swede_rutabaga,3301210500,swede_rutabaga,3301210500 Šalotka (jarná),Shallot (spring),onions,3301220400,onions,3301220400 -Kapusta hlávková s podsevom,Head cabbage with undersowing,white_cabbage,3301210212,white_cabbage,3301210212 \ No newline at end of file +Kapusta hlávková s podsevom,Head cabbage with undersowing,white_cabbage,3301210212,white_cabbage,3301210212 +Tritikale ozimné,Winter triticale,winter_triticale,3301010801,winter_triticale,3301010801 +Tritikale jarné,Spring triticale,spring_triticale,3301010802,spring_triticale,3301010802 +Zemiaky,Potatoes,potatoes,3301030000,potatoes,3301030000 +Zemiaky konzumné,Ware potatoes,potatoes,3301030000,potatoes,3301030000 +Ostrov biodiverzity,Biodiversity island,not_known_and_other,3399000000,not_known_and_other,3399000000 +Ostrov biodiverzity medonosný,Melliferous biodiversity island,not_known_and_other,3399000000,not_known_and_other,3399000000 +Biopás pre medonosné plodiny kosený po 31. júli,Melliferous field strip mown after 31 July,not_known_and_other,3399000000,not_known_and_other,3399000000 +Pôda ležiaca úhorom s produkciou s hnojením,Fallow with production and fertilisation,fallow_land_not_crop,3301110000,fallow_land_not_crop,3301110000 +Pôda ležiaca úhorom s produkciou bez hnojenia,Fallow with production without fertilisation,fallow_land_not_crop,3301110000,fallow_land_not_crop,3301110000 +Tekvica,Pumpkin,pumpkin_squash_gourd,3301140400,pumpkin_squash_gourd,3301140400 +Patizón,Pattypan squash,pumpkin_squash_gourd,3301140400,pumpkin_squash_gourd,3301140400 +Kapusta,Cabbage,brassica_oleracea_cabbage,3301210200,brassica_oleracea_cabbage,3301210200 +Kel,Savoy cabbage,savoy_cabbage,3301210211,savoy_cabbage,3301210211 +Kel hlávkový s podsevom,Savoy cabbage undersown,savoy_cabbage,3301210211,savoy_cabbage,3301210211 +Kapusta pekinská,Chinese cabbage,chinese_cabbage,3301210205,chinese_cabbage,3301210205 +Kapusta čínska,Chinese cabbage,chinese_cabbage,3301210205,chinese_cabbage,3301210205 +Kaleráb,Kohlrabi,kohlrabi,3301210209,kohlrabi,3301210209 +Brokolica,Broccoli,broccoli,3301210202,broccoli,3301210202 +Reďkev,Radish,radish,3301290600,radish,3301290600 +Reďkev siata (čierna a reďkovka),Radish (black and small),radish,3301290600,radish,3301290600 +Reďkovka,Small radish,radish,3301290600,radish,3301290600 +Paprika,Pepper,bell_pepper_paprika,3301300100,bell_pepper_paprika,3301300100 +Cibuľa,Onion,onions,3301220400,onions,3301220400 +Cesnak,Garlic,garlic,3301220200,garlic,3301220200 +Pór,Leek,leek,3301220300,leek,3301220300 +Repa obyčajná cviklová (cvikla),Beetroot,beetroot_beets,3301290200,beetroot_beets,3301290200 +Repa obyčajná (mangold),Chard,chard,3301310100,chard,3301310100 +Čučoriedka,Blueberry,blueberry,3303020400,blueberry,3303020400 +Mrkva,Carrot,carrots_daucus,3301290300,carrots_daucus,3301290300 +Paštrnák,Parsnip,parsnips,3301290500,parsnips,3301290500 +Zeler,Celery,celery,3301250000,celery,3301250000 +Rajčiaky,Tomato,tomato,3301280000,tomato,3301280000 +Baklažán,Aubergine,aubergine_eggplant,3301260000,aubergine_eggplant,3301260000 +Špenát,Spinach,spinach,3301310800,spinach,3301310800 +Šalát,Lettuce,salads_lettuce_leaf_vegetables,3301310000,salads_lettuce_leaf_vegetables,3301310000 +Štiavec,Sorrel,sorrel,3301310700,sorrel,3301310700 +Čakanka obyčajná,Chicory,chicory_chicories,3301310200,chicory_chicories,3301310200 +Melóny,Melons,melon,3301140300,melon,3301140300 +Dyňa,Watermelon,watermelon,3301140500,watermelon,3301140500 +Vinohrad so zlatým žltnutím,Vineyard with flavescence dorée,vineyards_wine_vine_rebland_grapes,3303060000,vineyards_wine_vine_rebland_grapes,3303060000 +Kukurica cukrová s podsevom,Sweet maize undersown,grain_maize_corn_popcorn,3301010600,grain_maize_corn_popcorn,3301010600 +Kukurica pukancová s podsevom,Popcorn maize undersown,grain_maize_corn_popcorn,3301010600,grain_maize_corn_popcorn,3301010600 +Senovka grécka,Fenugreek,fenugreek,3301020400,fenugreek,3301020400 +Cícer,Chickpea,chickpeas,3301020200,chickpeas,3301020200 +Fazuľa obyčajná,Common bean,beans,3301020100,beans,3301020100 +Fazuľa ostrolistá,Tepary bean,beans,3301020100,beans,3301020100 +Fazuľa šarlátová,Runner bean,beans,3301020100,beans,3301020100 +Fazuľa mesiacovitá,Lima bean,beans,3301020100,beans,3301020100 +Láskavec,Amaranth,amaranth,3301150100,amaranth,3301150100 +Komonica lekárska,Ribbed melilot,melilot,3301090304,melilot,3301090304 +Komonica biela,White melilot,melilot,3301090304,melilot,3301090304 +Jastrabina východná,Eastern galega,galega,3301081900,galega,3301081900 +Vtákonoha siata,Serradella,serradella,3301084200,serradella,3301084200 +Slez kŕmny,Fodder mallow,malva,3301082800,malva,3301082800 +Slez maurský,Moorish mallow,malva,3301082800,malva,3301082800 +Koriander siaty,Coriander,coriander,3301061215,coriander,3301061215 +Mäta,Mint,mints_peppermint,3301061222,mints_peppermint,3301061222 +Medovka,Lemon balm,lemon_balm_melissa,3301061220,lemon_balm_melissa,3301061220 +Ligurček lekársky,Lovage,lovage_maggiplant,3301061221,lovage_maggiplant,3301061221 +Bedrovník anízový,Anise,anise_aniseed,3301061205,anise_aniseed,3301061205 +Šalvia,Sage,sage_chia,3301190000,sage_chia,3301190000 +Borák lekársky,Borage,borage,3301061209,borage,3301061209 +Yzop lekársky,Hyssop,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299,other_aromatic_medicinal_culinary_plants_spices_herbs,3301061299 +Petržlen,Parsley,parsly,3301061227,parsly,3301061227 +Petržlen vňaťový kučeravý,Curly leaf parsley,parsly,3301061227,parsly,3301061227 +Kapucínka väčšia,Garden nasturtium,nasturtiums,3301061224,nasturtiums,3301061224 +Chren dedinský,Horseradish,horseradish,3301210400,horseradish,3301210400 +Povojník purpurový (batát),Sweet potato,sweet_potatoes,3301040000,sweet_potatoes,3301040000 +Povojník batátový,Sweet potato,sweet_potatoes,3301040000,sweet_potatoes,3301040000 +Slivka čerešňoplodá (myrobalán),Cherry plum,plums,3303011300,plums,3303011300 +Jarabina oskorušová,Service tree,rowan_rowanberries,3303021300,rowan_rowanberries,3303021300 +Topoľ chlpatoplodý,Poplar,populus,3306070000,populus,3306070000 +Topoľ simonov,Simon's poplar,populus,3306070000,populus,3306070000 +Vŕba rakytová,Goat willow,willows_osiers,3306080000,willows_osiers,3306080000 +Zalesnená orná pôda,Afforested arable land,afforestation_reforestation,3306010000,afforestation_reforestation,3306010000 diff --git a/tests/test_hcat_codes.py b/tests/test_hcat_codes.py index 4b0c6c16..60ba1d8d 100644 --- a/tests/test_hcat_codes.py +++ b/tests/test_hcat_codes.py @@ -2,34 +2,18 @@ A wrong code is invisible: the file validates, and a crop query quietly returns another crop. `tests/data-files/hcat3.csv` is the taxonomy, copied from -https://fiboa.org/code/hcat3.csv. +https://fiboa.org/code/hcat3.csv. `pixi run check-hcat` runs the same check +against the tables online. """ import csv +import os from pathlib import Path -DATA = Path(__file__).parent / "data-files" +from fiboa_cli.converters import Converters +from fiboa_cli.datasets.commons.hcat import AddHCATMixin -# Rows that are wrong today, recorded so a new mistake still fails. Each is -# tracked in https://github.com/fiboa/cli/issues/304; remove an entry when the -# table it comes from is fixed. Keyed by (name, code) because the same bad pair -# was copied into several tables. -KNOWN_BAD = { - # the typo is in the taxonomy, which spells it "parsly"; the tables are right - ("parsley", "3301061227"), - ("aspargus", "3301200000"), # taxonomy: asparagus - ("pistachios", "3303030400"), # taxonomy: pistachio - ("flax_fibre", "3301060701"), # taxonomy: flax_linen - # name and code are different crops: the codes belong to esparsette and sweet lupins - ("lupins", "3301020300"), - ("soybeans", "3301020700"), - # right name, wrong code - ("vineyards_wine_vine_rebland_grapes", "3303070000"), # 3303060000 - ("topinambur_jerusalem_artichoke", "3301290900"), # 3301180000 - ("iris", "3301082200"), # 3301082300 - ("lentils", "3301020300"), # 3301020500 - ("zucchini_courgette", "3301140200"), # 3301140600, already fixed upstream -} +DATA = Path(__file__).parent / "data-files" def read_rows(path): @@ -42,21 +26,52 @@ def read_rows(path): return [] +def row_key(row): + return (row.get("original_code") or "").strip() or (row.get("original_name") or "").strip() + + +def applied_tables(): + """Per fixture table a converter reads: the rows it applies, supplements winning by code.""" + converters = Converters() + tables, missing = {}, [] + for _id in converters.list_ids(): + converter = converters.load(_id) + if not isinstance(converter, AddHCATMixin) or not converter.hcat_mapping_csv: + continue + folder = DATA / "convert" / _id + main = folder / os.path.basename(converter.hcat_mapping_csv) + if not main.exists(): + continue + rows = {row_key(r): r for r in read_rows(main)} + for url in converter.hcat_mapping_supplements: + supplement = folder / os.path.basename(url) + if not supplement.exists(): + missing.append(str(supplement.relative_to(DATA))) + continue + rows |= {row_key(r): r for r in read_rows(supplement)} + tables[main] = list(rows.values()) + return tables, missing + + def test_mapping_tables_agree_with_the_taxonomy(): taxonomy = {r["HCAT3_name"]: r["HCAT3_code"] for r in read_rows(DATA / "hcat3.csv")} assert len(taxonomy) > 300, "the taxonomy fixture looks truncated" + applied, missing = applied_tables() + assert not missing, "supplements without a fixture: " + ", ".join(missing) + wrong = [] for path in sorted(DATA.glob("**/*.csv")): if path.name == "hcat3.csv": continue - rows = read_rows(path) + # a table a converter reads is checked as corrected by its supplements + rows = applied.get(path) or read_rows(path) if not rows or "HCAT3_name" not in rows[0] or "HCAT3_code" not in rows[0]: continue for row in rows: name = (row.get("HCAT3_name") or "").strip() code = (row.get("HCAT3_code") or "").strip() - if (not name and not code) or (name, code) in KNOWN_BAD: + if not name and not code: continue where = f"{path.relative_to(DATA)} {row.get('original_code')}" if name not in taxonomy: