From 35bbcdc2fa6f313d0895ff340906ef5aae1296a1 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:13:51 +0200 Subject: [PATCH 01/17] test(compliance): add portability rung M6, date and time scalars One dataset with YEAR and DATE_TRUNC fields and an AVG(DATEDIFF(...)) metric, using the part-first DATEDIFF(part, start, end) order from expression_language.md. Adds one construct on top of the baseline. Co-authored-by: Isaac --- .../portability/m6/m6_date_scalars.ossie.yaml | 59 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 ++++ 2 files changed, 69 insertions(+) create mode 100644 compliance/portability/m6/m6_date_scalars.ossie.yaml diff --git a/compliance/portability/m6/m6_date_scalars.ossie.yaml b/compliance/portability/m6/m6_date_scalars.ossie.yaml new file mode 100644 index 000000000..50e73736e --- /dev/null +++ b/compliance/portability/m6/m6_date_scalars.ossie.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M6 ladder rung: date and time scalars from the portable core (YEAR, +# DATE_TRUNC, DATEDIFF). DATEDIFF uses the spec's part-first argument order +# DATEDIFF(part, start, end). The date-part fields resolve inside their own +# dataset (sibling columns), and the DATEDIFF metric uses the qualified form. +version: "0.2.0.dev0" +name: m6_sales +description: "M6: date and time scalar functions from the portable core." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + datatype: Date + - name: ship_date + expression: + dialects: + - dialect: ANSI_SQL + expression: ship_date + datatype: Date + - name: order_year + expression: + dialects: + - dialect: ANSI_SQL + expression: YEAR(order_date) + datatype: Integer + - name: order_month_start + expression: + dialects: + - dialect: ANSI_SQL + expression: DATE_TRUNC('month', order_date) + datatype: Date +metrics: + - name: avg_fulfillment_days + expression: + dialects: + - dialect: ANSI_SQL + expression: AVG(DATEDIFF('day', sales.order_date, sales.ship_date)) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 0382f779f..21a9cd350 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -103,3 +103,13 @@ rungs: variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + + - id: M6 + description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) + element_under_test: + kind: fields and metric + name: order_year, order_month_start, avg_fulfillment_days + construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) + reference_form_primary: qualified # the fields reference sibling columns intra-dataset + variants: + - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From 8629e18efb2462f712c8208a10e0b42ab04d610d Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:14:20 +0200 Subject: [PATCH 02/17] test(compliance): add the FILTER (WHERE ...) form to portability rung M3 expression_language.md makes the aggregate FILTER (WHERE ...) clause REQUIRED. Two variants (qualified and bare) express the same filtered aggregate as the existing CASE WHEN fixtures, so a difference between the two forms isolates handling of the clause itself. Co-authored-by: Isaac --- .../m3/m3_filter_clause_bare.ossie.yaml | 47 +++++++++++++++++++ .../m3/m3_filter_clause_qualified.ossie.yaml | 47 +++++++++++++++++++ compliance/portability/manifest.yaml | 4 +- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 compliance/portability/m3/m3_filter_clause_bare.ossie.yaml create mode 100644 compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml diff --git a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml new file mode 100644 index 000000000..795743af6 --- /dev/null +++ b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M3 ladder rung, filter-clause form: the same filtered aggregate as the CASE +# WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that +# expression_language.md makes REQUIRED. Comparing the two forms shows whether a +# converter handles the clause itself. Variant: BARE reference form, kept as a sibling variant. +version: "0.2.0.dev0" +name: m3_sales +description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, bare reference form." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String +metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(amount) FILTER (WHERE status = 'completed') + datatype: Decimal diff --git a/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml new file mode 100644 index 000000000..49af6d340 --- /dev/null +++ b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml @@ -0,0 +1,47 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M3 ladder rung, filter-clause form: the same filtered aggregate as the CASE +# WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that +# expression_language.md makes REQUIRED. Comparing the two forms shows whether a +# converter handles the clause itself. Variant: QUALIFIED reference form, carried as primary. +version: "0.2.0.dev0" +name: m3_sales +description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, qualified reference form." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: status + expression: + dialects: + - dialect: ANSI_SQL + expression: status + datatype: String +metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) FILTER (WHERE sales.status = 'completed') + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 21a9cd350..965b0cd0d 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -94,7 +94,7 @@ rungs: - {id: M2.aggregates.bare, file: m2/m2_aggregates_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} - id: M3 - description: a filtered aggregate in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) + description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) and with the REQUIRED FILTER (WHERE ...) clause element_under_test: kind: metric name: completed_amount @@ -103,6 +103,8 @@ rungs: variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + - {id: M3.filter_clause.qualified, file: m3/m3_filter_clause_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: filter-clause-form} + - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form} - id: M6 description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) From 47a535f84eecbe26e7a712d7b516f347afbdbde5 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:14:42 +0200 Subject: [PATCH 03/17] test(compliance): add portability rung M4, a cross-dataset metric Two datasets joined by a declared relationship and one model-scoped metric that aggregates across the join (SUM over one dataset divided by a COUNT DISTINCT over the other), referencing fields as dataset.field. Co-authored-by: Isaac --- .../m4/m4_cross_dataset.ossie.yaml | 63 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 +++ 2 files changed, 73 insertions(+) create mode 100644 compliance/portability/m4/m4_cross_dataset.ossie.yaml diff --git a/compliance/portability/m4/m4_cross_dataset.ossie.yaml b/compliance/portability/m4/m4_cross_dataset.ossie.yaml new file mode 100644 index 000000000..768eb7901 --- /dev/null +++ b/compliance/portability/m4/m4_cross_dataset.ossie.yaml @@ -0,0 +1,63 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M4 ladder rung: two datasets and a relationship, with a metric that +# aggregates across the join. A cross-dataset metric must qualify its columns +# by dataset to be unambiguous, so this rung has a single qualified fixture +# (no bare variant: a bare column would not say which dataset it belongs to). +version: "0.2.0.dev0" +name: m4_sales +description: "M4: two datasets, a relationship, and a cross-dataset metric." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer + - name: customers + source: samples.sales.customers + primary_key: [customer_id] + fields: + - name: customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: customer_id + datatype: Integer +relationships: + - name: sales_to_customer + from: sales + to: customers + from_columns: [customer_id] + to_columns: [customer_id] +metrics: + - name: revenue_per_customer + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) / COUNT(DISTINCT customers.customer_id) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 965b0cd0d..dad7a0836 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -115,3 +115,13 @@ rungs: reference_form_primary: qualified # the fields reference sibling columns intra-dataset variants: - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} + + - id: M4 + description: two datasets joined by a relationship, with a model-scoped metric across the join + element_under_test: + kind: metric + name: revenue_per_customer + construct: cross-dataset (model-scoped) metric over a declared relationship + reference_form_primary: qualified # model-scoped metrics reference dataset.field + variants: + - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From b4b27cbd6eea6756a0380137f19e996037eebdb1 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:15:07 +0200 Subject: [PATCH 04/17] test(compliance): add portability rung M5, metric-to-metric composition A model-scoped metric (avg_order_value) composed from two other model-scoped metrics referenced by name, per Metric Scoping in core-spec/spec.md. Co-authored-by: Isaac --- .../portability/m5/m5_metric_ref.ossie.yaml | 59 +++++++++++++++++++ compliance/portability/manifest.yaml | 10 ++++ 2 files changed, 69 insertions(+) create mode 100644 compliance/portability/m5/m5_metric_ref.ossie.yaml diff --git a/compliance/portability/m5/m5_metric_ref.ossie.yaml b/compliance/portability/m5/m5_metric_ref.ossie.yaml new file mode 100644 index 000000000..29eda689d --- /dev/null +++ b/compliance/portability/m5/m5_metric_ref.ossie.yaml @@ -0,0 +1,59 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# M5 ladder rung: metric composition, a model-scoped metric that references +# other model-scoped metrics by name (avg_order_value refers to total_amount and +# order_count), as described under Metric Scoping in core-spec/spec.md, where a +# model-scoped metric is referenced as metric_name. Single fixture. +version: "0.2.0.dev0" +name: m5_sales +description: "M5: a metric that references other metrics (composition)." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: order_id + expression: + dialects: + - dialect: ANSI_SQL + expression: order_id + datatype: Integer +metrics: + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(sales.amount) + datatype: Decimal + - name: order_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(sales.order_id) + datatype: Integer + - name: avg_order_value + expression: + dialects: + - dialect: ANSI_SQL + expression: total_amount / order_count + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index dad7a0836..a4965ecc2 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -125,3 +125,13 @@ rungs: reference_form_primary: qualified # model-scoped metrics reference dataset.field variants: - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} + + - id: M5 + description: a model-scoped metric composed from other model-scoped metrics, referenced by name + element_under_test: + kind: metric + name: avg_order_value + construct: metric-to-metric reference (total_amount / order_count) + reference_form_primary: qualified # base metrics qualify dataset.field; the composite references metric names + variants: + - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} From 729aaa39832ae639cdbc4f1d077da53aba8402ea Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 22 Oct 2026 17:15:42 +0200 Subject: [PATCH 05/17] docs(compliance): describe portability rungs M0-M6 Lists the M3 FILTER (WHERE ...) form and rungs M4-M6 in the README table, notes that M7 and M8 wait on the window ORDER BY NULL-placement default, and records that the fixtures live in this repository. Co-authored-by: Isaac --- compliance/portability/README.md | 19 +++++++++++-------- compliance/portability/manifest.yaml | 25 ++++++++++++------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/compliance/portability/README.md b/compliance/portability/README.md index 9039231ac..7a87927f0 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -17,7 +17,7 @@ under the License. --> -# Portability measurement ladder (M0-M3) +# Portability measurement ladder (M0-M6) A small set of portable fixtures, and a grader, for measuring how faithfully a model survives a round-trip through each converter. This is issue #576 @@ -37,7 +37,10 @@ difference in a converter's output localizes to a single construct: | M0 | baseline | one dataset, one field, one `SUM` metric | | M1 | a scalar field, no aggregation | `amount * 1.1` calculated column | | M2 | the core aggregate set | `SUM`, `COUNT`, `COUNT(DISTINCT)`, `AVG`, `MIN`, `MAX` | -| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)` | +| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)`, and the same with `FILTER (WHERE ...)` | +| M4 | a cross-dataset metric | two datasets, a relationship, a model-scoped metric across the join | +| M5 | metric composition | a model-scoped metric built from two other metrics by name | +| M6 | date and time scalars | `YEAR`, `DATE_TRUNC`, `DATEDIFF(part, start, end)` | `manifest.yaml` lists the fixtures, the element each rung exercises, and the per-element outcome vocabulary the grader assigns (PORTABLE / TAG_PRESERVED, @@ -61,10 +64,10 @@ grades the real fixtures all-PORTABLE. See `grader/README.md`. ## Scope and placement -Rungs M4-M8 (relationships and cross-dataset metrics, metric-to-metric -composition, date and time scalars, window and offset functions) are held -pending #354, #591 and #587, and are not in this set. +Rungs M7 and M8 (window and offset functions: time intelligence and +ratio-to-total) are held until the default NULL placement in window `ORDER BY` +is settled (#587), and are not in this set. -Location is a starting point. If these should sit with the interoperability -harness from #348 or elsewhere once that lands, they can move; the fixtures are -self-contained and the manifest paths are relative. +The fixtures measure the specification rather than any one converter, so they +live in this repository; the fixtures are self-contained and the manifest paths +are relative. diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index a4965ecc2..3d8cd12b0 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -35,9 +35,8 @@ # expression, so it carries a single fixture. # # All fixtures validate.py-pass on apache/ossie main (9531429) with sqlglot -# 29.0.1. Rungs M4-M8 (relationships, metric composition, date scalars, window -# and offset functions) are held pending #354, #591 and #587; they are not in -# this set. +# 29.0.1. Rungs M7 and M8 (window and offset functions) are held pending #587; +# they are not in this set. outcomes: # per-element, assigned by the grader after diffing input vs returned output @@ -106,16 +105,6 @@ rungs: - {id: M3.filter_clause.qualified, file: m3/m3_filter_clause_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: filter-clause-form} - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form} - - id: M6 - description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) - element_under_test: - kind: fields and metric - name: order_year, order_month_start, avg_fulfillment_days - construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) - reference_form_primary: qualified # the fields reference sibling columns intra-dataset - variants: - - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - - id: M4 description: two datasets joined by a relationship, with a model-scoped metric across the join element_under_test: @@ -135,3 +124,13 @@ rungs: reference_form_primary: qualified # base metrics qualify dataset.field; the composite references metric names variants: - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} + + - id: M6 + description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) + element_under_test: + kind: fields and metric + name: order_year, order_month_start, avg_fulfillment_days + construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) + reference_form_primary: qualified # the fields reference sibling columns intra-dataset + variants: + - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From 9a6dcc722125f433d0ddd62b8be7d48cfc12c24b Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 30 Oct 2026 06:45:07 +0100 Subject: [PATCH 06/17] test(compliance): cover M4-M6 in the identity test, name every element by kind and name Follow-ups from the review of #631: - The identity regression test looped over M0-M3 only; it now covers M0-M6, and the grader README says so. - M5's reference_form_primary said "qualified", but its only variant (the composite under test) references metric names; it is now "metric-name". - M6 named "fields and metric" with a comma-joined name, and M2 named "metrics" without a name, so a lookup by kind and name found neither. Every rung now lists elements_under_test, each with one kind (metric or field) and one name. Two new tests keep the manifest honest: every listed element is found by kind and name in each variant of its rung, and each primary variant uses the rung's primary reference form. Both fail on the previous manifest. Co-authored-by: Isaac --- compliance/portability/grader/README.md | 6 +- compliance/portability/grader/test_grade.py | 54 ++++++++++++-- compliance/portability/manifest.yaml | 83 +++++++++++++-------- 3 files changed, 105 insertions(+), 38 deletions(-) diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index 0f699de97..cbd3351f9 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -62,5 +62,7 @@ PLACEHOLDER, ADDED, RELABELED, DROPPED_DIAGNOSED, DROPPED_SILENT, REJECTED. ## Tests `test_grade.py` has one test per outcome plus an identity check that runs the real -M0-M3 fixtures through a no-op converter and asserts every element grades -PORTABLE. Run with `PYTHONPATH=. pytest test_grade.py`. +M0-M6 fixtures through a no-op converter and asserts every element grades +PORTABLE. Two manifest checks assert that every element the manifest names is found +by kind and name in each variant, and that each primary variant uses the rung's primary +reference form. Run with `PYTHONPATH=. pytest test_grade.py`. diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 07e82f466..415d97c4d 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -18,8 +18,11 @@ """Outcome tests for the #576 fidelity grader. Each test builds a tiny input document and a synthetic returned document that -exercises exactly one outcome. The last test runs the real posted M0-M3 fixtures +exercises exactly one outcome. The identity test runs the real posted M0-M6 fixtures through a no-op converter: returned unchanged, every element must grade PORTABLE. +The manifest tests check that every element the manifest names can be found by +kind and name in each variant, and that each primary variant uses the rung's +primary reference form. """ import glob @@ -29,6 +32,8 @@ import grade +HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + def _metric(name, dialect, expression): return {"name": name, "expression": {"dialects": [{"dialect": dialect, "expression": expression}]}} @@ -291,16 +296,53 @@ def test_whitespace_is_not_a_difference(): assert _only(grade.grade(src, returned)).outcome == grade.PORTABLE -def test_identity_on_real_m0_m3_fixtures_is_all_portable(): - here = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +def test_identity_on_real_m0_m6_fixtures_is_all_portable(): files = sorted( path - for rung in ("m0", "m1", "m2", "m3") - for path in glob.glob(os.path.join(here, rung, "*.ossie.yaml")) + for rung in ("m0", "m1", "m2", "m3", "m4", "m5", "m6") + for path in glob.glob(os.path.join(HERE, rung, "*.ossie.yaml")) ) - assert files, "no M0-M3 fixtures found" + assert files, "no M0-M6 fixtures found" for path in files: document = grade._load(path) report = grade.grade(document, document) bad = [(f.name, f.outcome) for f in report.findings if f.outcome != grade.PORTABLE] assert not bad, f"{os.path.basename(path)} graded non-PORTABLE: {bad}" + + +def _manifest_rungs(): + return grade._load(os.path.join(HERE, "manifest.yaml"))["rungs"] + + +def _names(document, kind): + if kind == "metric": + return {metric["name"] for metric in document.get("metrics") or []} + return { + field["name"] + for dataset in document.get("datasets") or [] + for field in dataset.get("fields") or [] + } + + +def test_every_manifest_element_is_found_by_kind_and_name_in_each_variant(): + checked = 0 + for rung in _manifest_rungs(): + for variant in rung["variants"]: + document = grade._load(os.path.join(HERE, variant["file"])) + for element in rung["elements_under_test"]: + assert element["kind"] in ("metric", "field"), (rung["id"], element) + assert element["name"] in _names(document, element["kind"]), ( + variant["id"], + element, + ) + checked += 1 + assert checked, "the manifest names no elements" + + +def test_each_primary_variant_uses_the_rungs_primary_reference_form(): + for rung in _manifest_rungs(): + if rung["reference_form_primary"] == "n/a": + continue + primary = [v for v in rung["variants"] if v["role"] == "primary"] + assert len(primary) == 1, rung["id"] + assert primary[0]["reference_form"] == rung["reference_form_primary"], rung["id"] diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 3d8cd12b0..9c0465903 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -23,7 +23,8 @@ # through every converter; a vendor-tagged input would only exercise one # converter, which is not what a cross-converter measurement wants. # -# This manifest lists the fixtures and the single element each rung exercises. +# This manifest lists the fixtures and the elements each rung exercises, each by +# kind (metric or field) and name, as they appear in every variant of the rung. # It does NOT encode per-converter expected outputs: those are measured by a # grader that diffs the returned document against the input element by element # into the outcome vocabulary below. Conformance stays a floor (validate.py @@ -62,10 +63,10 @@ outcomes: rungs: - id: M0 description: one dataset, one field, one SUM metric - element_under_test: - kind: metric - name: total_amount - construct: single-column aggregate (SUM) + elements_under_test: + - kind: metric + name: total_amount + construct: single-column aggregate (SUM) reference_form_primary: qualified variants: - {id: M0.ansi.qualified, file: m0/m0_ansi_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} @@ -74,19 +75,35 @@ rungs: - id: M1 description: baseline plus one scalar calculated-column field expression, no aggregation - element_under_test: - kind: field - name: amount_with_tax - construct: scalar field expression (amount * 1.1) + elements_under_test: + - kind: field + name: amount_with_tax + construct: scalar field expression (amount * 1.1) reference_form_primary: n/a # intra-dataset field expression references its sibling column bare variants: - {id: M1.scalar_field, file: m1/m1_scalar_field.ossie.yaml, tag: ANSI_SQL, reference_form: intra-dataset, role: primary} - id: M2 description: the core aggregate set (SUM, COUNT, COUNT DISTINCT, AVG, MIN, MAX) - element_under_test: - kind: metrics - construct: core aggregate set + elements_under_test: + - kind: metric + name: sum_amount + construct: SUM + - kind: metric + name: count_orders + construct: COUNT + - kind: metric + name: distinct_customers + construct: COUNT DISTINCT + - kind: metric + name: avg_amount + construct: AVG + - kind: metric + name: min_amount + construct: MIN + - kind: metric + name: max_amount + construct: MAX reference_form_primary: qualified variants: - {id: M2.aggregates.qualified, file: m2/m2_aggregates_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} @@ -94,10 +111,10 @@ rungs: - id: M3 description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) and with the REQUIRED FILTER (WHERE ...) clause - element_under_test: - kind: metric - name: completed_amount - construct: conditional aggregation + elements_under_test: + - kind: metric + name: completed_amount + construct: conditional aggregation reference_form_primary: qualified variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} @@ -107,30 +124,36 @@ rungs: - id: M4 description: two datasets joined by a relationship, with a model-scoped metric across the join - element_under_test: - kind: metric - name: revenue_per_customer - construct: cross-dataset (model-scoped) metric over a declared relationship + elements_under_test: + - kind: metric + name: revenue_per_customer + construct: cross-dataset (model-scoped) metric over a declared relationship reference_form_primary: qualified # model-scoped metrics reference dataset.field variants: - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - id: M5 description: a model-scoped metric composed from other model-scoped metrics, referenced by name - element_under_test: - kind: metric - name: avg_order_value - construct: metric-to-metric reference (total_amount / order_count) - reference_form_primary: qualified # base metrics qualify dataset.field; the composite references metric names + elements_under_test: + - kind: metric + name: avg_order_value + construct: metric-to-metric reference (total_amount / order_count) + reference_form_primary: metric-name # the composite under test references metric names (its base metrics qualify dataset.field) variants: - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} - id: M6 description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) - element_under_test: - kind: fields and metric - name: order_year, order_month_start, avg_fulfillment_days - construct: date scalar functions (DATEDIFF(part, start, end) per expression_language.md) - reference_form_primary: qualified # the fields reference sibling columns intra-dataset + elements_under_test: + - kind: field + name: order_year + construct: YEAR + - kind: field + name: order_month_start + construct: DATE_TRUNC + - kind: metric + name: avg_fulfillment_days + construct: DATEDIFF(part, start, end) per expression_language.md + reference_form_primary: qualified # the metric qualifies dataset.field; the fields reference sibling columns bare (intra-dataset) variants: - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} From 047a052599102870851125ce068760b8bc0b1311 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 30 Oct 2026 06:48:50 +0100 Subject: [PATCH 07/17] test(compliance): carry M5's composition inlined, as the spec requires A metric expression cannot refer to other metrics (core-spec/spec.md, Metric Scoping; expression_language.md), and since #608 validate.py warns when one does. M5's composite, avg_order_value = total_amount / order_count, referenced the two metrics by name. The "Referenced as metric_name" row in the Metric Scoping table is how a query addresses a metric, not what an expression may contain. M5 now carries the composition the way a converter has to emit it when the source tool composes measures: the parts inlined, over qualified dataset.field references, SUM(sales.amount) / NULLIF(COUNT(sales.order_id), 0), NULL when there are no orders. The two base metrics stay alongside it. The fixture validates with no warnings on main's validator, and its primary reference form is qualified again. Co-authored-by: Isaac --- compliance/portability/README.md | 2 +- compliance/portability/m5/m5_metric_ref.ossie.yaml | 14 ++++++++------ compliance/portability/manifest.yaml | 8 ++++---- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/compliance/portability/README.md b/compliance/portability/README.md index 7a87927f0..dce66f612 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -39,7 +39,7 @@ difference in a converter's output localizes to a single construct: | M2 | the core aggregate set | `SUM`, `COUNT`, `COUNT(DISTINCT)`, `AVG`, `MIN`, `MAX` | | M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)`, and the same with `FILTER (WHERE ...)` | | M4 | a cross-dataset metric | two datasets, a relationship, a model-scoped metric across the join | -| M5 | metric composition | a model-scoped metric built from two other metrics by name | +| M5 | metric composition | a model-scoped metric built from two others, inlined (a metric expression cannot refer to other metrics) | | M6 | date and time scalars | `YEAR`, `DATE_TRUNC`, `DATEDIFF(part, start, end)` | `manifest.yaml` lists the fixtures, the element each rung exercises, and the diff --git a/compliance/portability/m5/m5_metric_ref.ossie.yaml b/compliance/portability/m5/m5_metric_ref.ossie.yaml index 29eda689d..030624c33 100644 --- a/compliance/portability/m5/m5_metric_ref.ossie.yaml +++ b/compliance/portability/m5/m5_metric_ref.ossie.yaml @@ -15,13 +15,15 @@ # specific language governing permissions and limitations # under the License. -# M5 ladder rung: metric composition, a model-scoped metric that references -# other model-scoped metrics by name (avg_order_value refers to total_amount and -# order_count), as described under Metric Scoping in core-spec/spec.md, where a -# model-scoped metric is referenced as metric_name. Single fixture. +# M5 ladder rung: metric composition. A metric expression cannot refer to other +# metrics (core-spec/spec.md, Metric Scoping; expression_language.md), so a +# composed measure in a source tool (avg_order_value = total_amount / order_count) +# is carried with its parts inlined: a ratio of two aggregates over qualified +# dataset.field references, NULL when there are no orders. The two base metrics +# stay alongside it. Single fixture. version: "0.2.0.dev0" name: m5_sales -description: "M5: a metric that references other metrics (composition)." +description: "M5: a composed metric, with the metrics it is built from inlined." datasets: - name: sales source: samples.sales.orders @@ -55,5 +57,5 @@ metrics: expression: dialects: - dialect: ANSI_SQL - expression: total_amount / order_count + expression: SUM(sales.amount) / NULLIF(COUNT(sales.order_id), 0) datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 9c0465903..0a0d09b75 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -133,14 +133,14 @@ rungs: - {id: M4.cross_dataset, file: m4/m4_cross_dataset.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - id: M5 - description: a model-scoped metric composed from other model-scoped metrics, referenced by name + description: a model-scoped metric composed from two others, with their expressions inlined elements_under_test: - kind: metric name: avg_order_value - construct: metric-to-metric reference (total_amount / order_count) - reference_form_primary: metric-name # the composite under test references metric names (its base metrics qualify dataset.field) + construct: composition inlined (total_amount / order_count as SUM(...) / NULLIF(COUNT(...), 0)); metric expressions cannot refer to other metrics + reference_form_primary: qualified variants: - - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: metric-name, role: primary} + - {id: M5.metric_ref, file: m5/m5_metric_ref.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - id: M6 description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) From c90a80d013c0022b8154b7c4449e9caec940f45b Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Sun, 1 Nov 2026 07:24:53 +0100 Subject: [PATCH 08/17] test(compliance): write M3's CASE form without ELSE 0, as the FILTER form means expression_language.md (Conditional Aggregations) states that FILTER (WHERE ...) is equivalent to a CASE with no ELSE for an aggregate that ignores NULL, and that an ELSE 0 "would count as a real row". M3's CASE fixtures wrote ELSE 0, so on rows that exist but do not match they give 0 where the FILTER variants added here give NULL; they were not the same aggregate the rung describes. Found by idk-arsh's execution check on apache/ossie-converters#24. Both CASE fixtures now read SUM(CASE WHEN ... THEN amount END), which also matches DAX (CALCULATE over no rows is BLANK). All four M3 fixtures validate on main. Co-authored-by: Isaac --- compliance/portability/README.md | 2 +- compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml | 6 ++++-- .../portability/m3/m3_filtered_agg_qualified.ossie.yaml | 6 ++++-- compliance/portability/manifest.yaml | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/compliance/portability/README.md b/compliance/portability/README.md index dce66f612..c8a8f4c99 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -37,7 +37,7 @@ difference in a converter's output localizes to a single construct: | M0 | baseline | one dataset, one field, one `SUM` metric | | M1 | a scalar field, no aggregation | `amount * 1.1` calculated column | | M2 | the core aggregate set | `SUM`, `COUNT`, `COUNT(DISTINCT)`, `AVG`, `MIN`, `MAX` | -| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x ELSE 0 END)`, and the same with `FILTER (WHERE ...)` | +| M3 | a filtered aggregate | `SUM(CASE WHEN ... THEN x END)` (no `ELSE`, so it equals the `FILTER` form), and the same with `FILTER (WHERE ...)` | | M4 | a cross-dataset metric | two datasets, a relationship, a model-scoped metric across the join | | M5 | metric composition | a model-scoped metric built from two others, inlined (a metric expression cannot refer to other metrics) | | M6 | date and time scalars | `YEAR`, `DATE_TRUNC`, `DATEDIFF(part, start, end)` | diff --git a/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml b/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml index c248c8f15..3c702b9b5 100644 --- a/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml +++ b/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml @@ -16,7 +16,9 @@ # under the License. # M3 ladder rung: a filtered aggregate in the spec's portable form -# SUM(CASE WHEN ... THEN x ELSE 0 END). Variant: BARE reference form, kept as a +# SUM(CASE WHEN ... THEN x END), with no ELSE: a non-matching row contributes +# NULL, so the result equals the FILTER (WHERE ...) form (expression_language.md, +# Conditional Aggregations; an ELSE 0 would count as a real row). Variant: BARE reference form, kept as a # sibling variant pending #343. version: "0.3.0.dev0" name: m3_sales @@ -42,5 +44,5 @@ metrics: expression: dialects: - dialect: ANSI_SQL - expression: SUM(CASE WHEN status = 'completed' THEN amount ELSE 0 END) + expression: SUM(CASE WHEN status = 'completed' THEN amount END) datatype: Decimal diff --git a/compliance/portability/m3/m3_filtered_agg_qualified.ossie.yaml b/compliance/portability/m3/m3_filtered_agg_qualified.ossie.yaml index 49fd711d4..61d4d2d9a 100644 --- a/compliance/portability/m3/m3_filtered_agg_qualified.ossie.yaml +++ b/compliance/portability/m3/m3_filtered_agg_qualified.ossie.yaml @@ -16,7 +16,9 @@ # under the License. # M3 ladder rung: a filtered aggregate in the spec's portable form -# SUM(CASE WHEN ... THEN x ELSE 0 END). Variant: QUALIFIED reference form, +# SUM(CASE WHEN ... THEN x END), with no ELSE: a non-matching row contributes +# NULL, so the result equals the FILTER (WHERE ...) form (expression_language.md, +# Conditional Aggregations; an ELSE 0 would count as a real row). Variant: QUALIFIED reference form, # carried as primary per #486. version: "0.3.0.dev0" name: m3_sales @@ -42,5 +44,5 @@ metrics: expression: dialects: - dialect: ANSI_SQL - expression: SUM(CASE WHEN sales.status = 'completed' THEN sales.amount ELSE 0 END) + expression: SUM(CASE WHEN sales.status = 'completed' THEN sales.amount END) datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 0a0d09b75..978e4bd5d 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -110,7 +110,7 @@ rungs: - {id: M2.aggregates.bare, file: m2/m2_aggregates_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} - id: M3 - description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x ELSE 0 END) and with the REQUIRED FILTER (WHERE ...) clause + description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x END) (no ELSE, equal to FILTER per expression_language.md) and with the REQUIRED FILTER (WHERE ...) clause elements_under_test: - kind: metric name: completed_amount From c56a29e816fa9e479c3f5b8565aaa781f2ceefe4 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 30 Oct 2026 07:02:16 +0100 Subject: [PATCH 09/17] test(compliance): carry the bare variants as dataset-scoped metrics, grade them Since #486 a model-scoped metric must reference fields as dataset.field, and since #608 validate.py warns on the bare variants of M0, M2 and M3 ("is not qualified"). Metric Scoping in core-spec/spec.md gives the bare form its place: a dataset-scoped metric (datasets[].metrics) references its own dataset's fields by single-part name. The four bare fixtures now carry their metrics there, and validate without warnings on main. The grader read only the model-root metrics, so a dataset-scoped metric was not graded at all, and the identity test still passed. It now collects datasets[].metrics too, keyed dataset.metric as the spec references them, and the identity test asserts that every metric in a fixture is graded. New tests: a dataset-scoped metric graded PORTABLE, dropped, and moved to the model (REWRITTEN, identity not preserved). All four fail without the grader change. Co-authored-by: Isaac --- compliance/portability/grader/README.md | 5 ++ compliance/portability/grader/grade.py | 16 ++-- compliance/portability/grader/test_grade.py | 54 ++++++++++++- .../portability/m0/m0_sum_bare.ossie.yaml | 26 +++--- .../m2/m2_aggregates_bare.ossie.yaml | 81 ++++++++++--------- .../m3/m3_filter_clause_bare.ossie.yaml | 20 +++-- .../m3/m3_filtered_agg_bare.ossie.yaml | 21 ++--- compliance/portability/manifest.yaml | 9 ++- 8 files changed, 152 insertions(+), 80 deletions(-) diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index cbd3351f9..020bb87b7 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -29,6 +29,11 @@ in `../manifest.yaml`. A thin runner (modelled on `cube/tools/interop_matrix.py` apache/ossie-converters) drives each converter and feeds the grader, so the measurement table becomes one command. +Elements are metrics and field expressions. A model-scoped metric is keyed by its +name, a dataset-scoped metric (`datasets[].metrics`) and a field by `dataset.name`, +the way the spec references them. A metric a converter moves between the two scopes +with the same expression is graded REWRITTEN (identity not preserved), not PORTABLE. + ## Usage ``` diff --git a/compliance/portability/grader/grade.py b/compliance/portability/grader/grade.py index 2074dfcaf..800b93080 100644 --- a/compliance/portability/grader/grade.py +++ b/compliance/portability/grader/grade.py @@ -174,7 +174,12 @@ class Element: def _collect(document): - """Collect metrics and field expressions, keyed by (kind, name).""" + """Collect metrics and field expressions, keyed by (kind, name). + + A dataset-scoped metric (``datasets[].metrics``) is keyed ``dataset.metric``, + the name the spec references it by, so it cannot collide with a model-scoped + metric of the same name. + """ elements = {} if not isinstance(document, dict): return elements @@ -192,10 +197,11 @@ def _collect(document): for field_def in dataset.get("fields") or []: if isinstance(field_def, dict) and field_def.get("name"): name = f"{dataset_name}.{field_def['name']}" - expression = field_def.get("expression") - elements[("field", name)] = Element( - "field", name, _dialect_map(expression) - ) + elements[("field", name)] = Element("field", name, _dialect_map(field_def.get("expression"))) + for metric in dataset.get("metrics") or []: + if isinstance(metric, dict) and metric.get("name"): + name = f"{dataset_name}.{metric['name']}" + elements[("metric", name)] = Element("metric", name, _dialect_map(metric.get("expression"))) return elements diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 415d97c4d..d04dfb381 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -22,7 +22,8 @@ through a no-op converter: returned unchanged, every element must grade PORTABLE. The manifest tests check that every element the manifest names can be found by kind and name in each variant, and that each primary variant uses the rung's -primary reference form. +primary reference form. The dataset-scoped tests check that a metric in +``datasets[].metrics`` is graded as ``dataset.metric``. """ import glob @@ -308,6 +309,10 @@ def test_identity_on_real_m0_m6_fixtures_is_all_portable(): report = grade.grade(document, document) bad = [(f.name, f.outcome) for f in report.findings if f.outcome != grade.PORTABLE] assert not bad, f"{os.path.basename(path)} graded non-PORTABLE: {bad}" + graded = sum(1 for f in report.findings if f.kind == "metric") + assert graded == len(_names(document, "metric")), ( + f"{os.path.basename(path)}: not every metric was graded" + ) def _manifest_rungs(): @@ -316,7 +321,11 @@ def _manifest_rungs(): def _names(document, kind): if kind == "metric": - return {metric["name"] for metric in document.get("metrics") or []} + return {metric["name"] for metric in document.get("metrics") or []} | { + metric["name"] + for dataset in document.get("datasets") or [] + for metric in dataset.get("metrics") or [] + } return { field["name"] for dataset in document.get("datasets") or [] @@ -346,3 +355,44 @@ def test_each_primary_variant_uses_the_rungs_primary_reference_form(): primary = [v for v in rung["variants"] if v["role"] == "primary"] assert len(primary) == 1, rung["id"] assert primary[0]["reference_form"] == rung["reference_form_primary"], rung["id"] + + +def _dataset_metric_document(metric_expression="SUM(amount)"): + return { + "datasets": [ + { + "name": "sales", + "fields": [{"name": "amount", "expression": {"dialects": [ + {"dialect": "ANSI_SQL", "expression": "amount"}]}}], + "metrics": [_metric("total_amount", "ANSI_SQL", metric_expression)], + } + ] + } + + +def test_a_dataset_scoped_metric_is_graded_under_dataset_dot_name(): + document = _dataset_metric_document() + report = grade.grade(document, document) + outcomes = {(f.kind, f.name): f.outcome for f in report.findings} + assert outcomes[("metric", "sales.total_amount")] == grade.PORTABLE + + +def test_a_dropped_dataset_scoped_metric_is_reported(): + returned = _dataset_metric_document() + returned["datasets"][0]["metrics"] = [] + report = grade.grade(_dataset_metric_document(), returned) + outcomes = {(f.kind, f.name): f.outcome for f in report.findings} + assert outcomes[("metric", "sales.total_amount")] == grade.DROPPED_SILENT + + +def test_a_dataset_scoped_metric_moved_to_the_model_is_not_the_same_element(): + returned = _dataset_metric_document() + returned["datasets"][0]["metrics"] = [] + returned["metrics"] = [_metric("total_amount", "ANSI_SQL", "SUM(amount)")] + report = grade.grade(_dataset_metric_document(), returned) + outcomes = {(f.kind, f.name): f.outcome for f in report.findings} + # Same expression under another name: one element whose identity changed, + # not a drop plus an unrelated addition. + assert outcomes[("metric", "sales.total_amount")] == grade.REWRITTEN + assert ("metric", "total_amount") not in outcomes + diff --git a/compliance/portability/m0/m0_sum_bare.ossie.yaml b/compliance/portability/m0/m0_sum_bare.ossie.yaml index 1b5df30d4..d45a111a6 100644 --- a/compliance/portability/m0/m0_sum_bare.ossie.yaml +++ b/compliance/portability/m0/m0_sum_bare.ossie.yaml @@ -18,13 +18,11 @@ # yaml-language-server: $schema=../../../../core-spec/ossie-schema.json # # M0 ladder rung -- one dataset, one field, one SUM metric. -# Variant: BARE reference form -- SUM(amount), a bare column name rather than -# dataset.field. ANSI_SQL tag. This is the "SUM(amount)" variant from the runs. -# Kept as a sibling variant to the qualified primary, not baked into the rung, -# because #343 (bare name means a source column inside a dataset-scoped metric) -# and #486 (qualified dataset.field only) are still settling which form is the -# model-level reference. sigma drops this as a cross-dataset metric on a -# one-dataset model. +# Variant: BARE reference form, carried as a DATASET-SCOPED metric +# (datasets[].metrics), where core-spec/spec.md (Metric Scoping) requires the +# single-part field name. A bare name in a model-scoped metric is not valid +# since #486; the qualified primary keeps the model-scoped form. +# ANSI_SQL tag. This is the "SUM(amount)" variant from the runs. version: "0.3.0.dev0" name: m0_sales description: M0 baseline -- single dataset, single field, single SUM metric. @@ -38,10 +36,10 @@ datasets: - dialect: ANSI_SQL expression: amount datatype: Decimal -metrics: - - name: total_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(amount) - datatype: Decimal + metrics: + - name: total_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(amount) + datatype: Decimal diff --git a/compliance/portability/m2/m2_aggregates_bare.ossie.yaml b/compliance/portability/m2/m2_aggregates_bare.ossie.yaml index c2ecd6133..6458f4305 100644 --- a/compliance/portability/m2/m2_aggregates_bare.ossie.yaml +++ b/compliance/portability/m2/m2_aggregates_bare.ossie.yaml @@ -15,8 +15,11 @@ # specific language governing permissions and limitations # under the License. -# M2 ladder rung: the core aggregate set as metrics. Variant: BARE reference -# form (column name only), kept as a sibling variant pending #343. +# M2 ladder rung: the core aggregate set as metrics. +# Variant: BARE reference form, carried as a DATASET-SCOPED metric +# (datasets[].metrics), where core-spec/spec.md (Metric Scoping) requires the +# single-part field name. A bare name in a model-scoped metric is not valid +# since #486; the qualified primary keeps the model-scoped form. version: "0.3.0.dev0" name: m2_sales description: "M2: core aggregate set, bare reference form." @@ -42,40 +45,40 @@ datasets: - dialect: ANSI_SQL expression: customer_id datatype: Integer -metrics: - - name: sum_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(amount) - datatype: Decimal - - name: count_orders - expression: - dialects: - - dialect: ANSI_SQL - expression: COUNT(order_id) - datatype: Integer - - name: distinct_customers - expression: - dialects: - - dialect: ANSI_SQL - expression: COUNT(DISTINCT customer_id) - datatype: Integer - - name: avg_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: AVG(amount) - datatype: Decimal - - name: min_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: MIN(amount) - datatype: Decimal - - name: max_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: MAX(amount) - datatype: Decimal + metrics: + - name: sum_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(amount) + datatype: Decimal + - name: count_orders + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(order_id) + datatype: Integer + - name: distinct_customers + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT customer_id) + datatype: Integer + - name: avg_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: AVG(amount) + datatype: Decimal + - name: min_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: MIN(amount) + datatype: Decimal + - name: max_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: MAX(amount) + datatype: Decimal diff --git a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml index 795743af6..25c34536f 100644 --- a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml +++ b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml @@ -18,7 +18,11 @@ # M3 ladder rung, filter-clause form: the same filtered aggregate as the CASE # WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that # expression_language.md makes REQUIRED. Comparing the two forms shows whether a -# converter handles the clause itself. Variant: BARE reference form, kept as a sibling variant. +# converter handles the clause itself. +# Variant: BARE reference form, carried as a DATASET-SCOPED metric +# (datasets[].metrics), where core-spec/spec.md (Metric Scoping) requires the +# single-part field name. A bare name in a model-scoped metric is not valid +# since #486; the qualified primary keeps the model-scoped form. version: "0.2.0.dev0" name: m3_sales description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, bare reference form." @@ -38,10 +42,10 @@ datasets: - dialect: ANSI_SQL expression: status datatype: String -metrics: - - name: completed_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(amount) FILTER (WHERE status = 'completed') - datatype: Decimal + metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(amount) FILTER (WHERE status = 'completed') + datatype: Decimal diff --git a/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml b/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml index 3c702b9b5..7f8674009 100644 --- a/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml +++ b/compliance/portability/m3/m3_filtered_agg_bare.ossie.yaml @@ -18,8 +18,11 @@ # M3 ladder rung: a filtered aggregate in the spec's portable form # SUM(CASE WHEN ... THEN x END), with no ELSE: a non-matching row contributes # NULL, so the result equals the FILTER (WHERE ...) form (expression_language.md, -# Conditional Aggregations; an ELSE 0 would count as a real row). Variant: BARE reference form, kept as a -# sibling variant pending #343. +# Conditional Aggregations; an ELSE 0 would count as a real row). +# Variant: BARE reference form, carried as a DATASET-SCOPED metric +# (datasets[].metrics), where core-spec/spec.md (Metric Scoping) requires the +# single-part field name. A bare name in a model-scoped metric is not valid +# since #486; the qualified primary keeps the model-scoped form. version: "0.3.0.dev0" name: m3_sales description: "M3: conditional aggregation, bare reference form." @@ -39,10 +42,10 @@ datasets: - dialect: ANSI_SQL expression: status datatype: String -metrics: - - name: completed_amount - expression: - dialects: - - dialect: ANSI_SQL - expression: SUM(CASE WHEN status = 'completed' THEN amount END) - datatype: Decimal + metrics: + - name: completed_amount + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(CASE WHEN status = 'completed' THEN amount END) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 978e4bd5d..8ac45085e 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -31,9 +31,12 @@ # passing); scoring stays a harness job (see #348). # # Reference form: the qualified dataset.field form is carried as the primary -# fixture wherever a metric references a dataset field, with the bare form as a -# sibling variant, pending #486 and #343. M1's element is an intra-dataset field -# expression, so it carries a single fixture. +# fixture wherever a metric references a dataset field, as a model-scoped metric. +# The bare form is a sibling variant carried as a dataset-scoped metric +# (datasets[].metrics), the only place the spec allows a single-part field name +# in a metric (Metric Scoping, #486). The grader keys a dataset-scoped metric as +# dataset.metric. M1's element is an intra-dataset field expression, so it +# carries a single fixture. # # All fixtures validate.py-pass on apache/ossie main (9531429) with sqlglot # 29.0.1. Rungs M7 and M8 (window and offset functions) are held pending #587; From f3296ad31627a964958a91c481a9e68ab153ad7b Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Mon, 2 Nov 2026 07:22:16 +0100 Subject: [PATCH 10/17] test(compliance): name manifest elements by the grader's keys; spec spelling in M6 Follow-ups from the second review of #631: - The manifest named fields bare (amount_with_tax, order_year, order_month_start), but the grader reports a field as dataset.field, so matching manifest elements to findings found no result for M1 or M6. Fields are now named sales., and a variant that carries the rung's metrics as dataset-scoped metrics says so with metrics_in: sales (the grader reports those as sales.). The manifest test now uses the grader's own _collect keys instead of building a name set of its own, so the two cannot drift apart; it fails on the previous manifest. - M6 wrote DATEDIFF('day', ...), the Tableau spelling in the mapping table; the spec writes the part as a keyword, DATEDIFF(day, ...). Co-authored-by: Isaac --- compliance/portability/grader/test_grade.py | 21 ++++++++----------- .../portability/m6/m6_date_scalars.ossie.yaml | 2 +- compliance/portability/manifest.yaml | 19 ++++++++++------- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index d04dfb381..5e1a7b481 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -320,17 +320,14 @@ def _manifest_rungs(): def _names(document, kind): - if kind == "metric": - return {metric["name"] for metric in document.get("metrics") or []} | { - metric["name"] - for dataset in document.get("datasets") or [] - for metric in dataset.get("metrics") or [] - } - return { - field["name"] - for dataset in document.get("datasets") or [] - for field in dataset.get("fields") or [] - } + # The grader's own keys, so the manifest and the findings cannot drift apart. + return {name for (element_kind, name) in grade._collect(document) if element_kind == kind} + + +def _reported_name(element, variant): + if element["kind"] == "metric" and variant.get("metrics_in"): + return f"{variant['metrics_in']}.{element['name']}" + return element["name"] def test_every_manifest_element_is_found_by_kind_and_name_in_each_variant(): @@ -340,7 +337,7 @@ def test_every_manifest_element_is_found_by_kind_and_name_in_each_variant(): document = grade._load(os.path.join(HERE, variant["file"])) for element in rung["elements_under_test"]: assert element["kind"] in ("metric", "field"), (rung["id"], element) - assert element["name"] in _names(document, element["kind"]), ( + assert _reported_name(element, variant) in _names(document, element["kind"]), ( variant["id"], element, ) diff --git a/compliance/portability/m6/m6_date_scalars.ossie.yaml b/compliance/portability/m6/m6_date_scalars.ossie.yaml index 50e73736e..8e160493c 100644 --- a/compliance/portability/m6/m6_date_scalars.ossie.yaml +++ b/compliance/portability/m6/m6_date_scalars.ossie.yaml @@ -55,5 +55,5 @@ metrics: expression: dialects: - dialect: ANSI_SQL - expression: AVG(DATEDIFF('day', sales.order_date, sales.ship_date)) + expression: AVG(DATEDIFF(day, sales.order_date, sales.ship_date)) datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 8ac45085e..dd37d3b1c 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -24,7 +24,10 @@ # converter, which is not what a cross-converter measurement wants. # # This manifest lists the fixtures and the elements each rung exercises, each by -# kind (metric or field) and name, as they appear in every variant of the rung. +# kind (metric or field) and by the name the grader reports it under: a field as +# dataset.field, a model-scoped metric by its name. A variant with +# `metrics_in: ` carries the rung's metrics as dataset-scoped metrics of +# that dataset, which the grader reports as .. # It does NOT encode per-converter expected outputs: those are measured by a # grader that diffs the returned document against the input element by element # into the outcome vocabulary below. Conformance stays a floor (validate.py @@ -74,13 +77,13 @@ rungs: variants: - {id: M0.ansi.qualified, file: m0/m0_ansi_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - {id: M0.ossiesql.qualified, file: m0/m0_ossiesql_qualified.ossie.yaml, tag: OSSIE_SQL_2026, reference_form: qualified, role: tag-sensitivity} - - {id: M0.sum.bare, file: m0/m0_sum_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + - {id: M0.sum.bare, file: m0/m0_sum_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant, metrics_in: sales} - id: M1 description: baseline plus one scalar calculated-column field expression, no aggregation elements_under_test: - kind: field - name: amount_with_tax + name: sales.amount_with_tax construct: scalar field expression (amount * 1.1) reference_form_primary: n/a # intra-dataset field expression references its sibling column bare variants: @@ -110,7 +113,7 @@ rungs: reference_form_primary: qualified variants: - {id: M2.aggregates.qualified, file: m2/m2_aggregates_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - - {id: M2.aggregates.bare, file: m2/m2_aggregates_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + - {id: M2.aggregates.bare, file: m2/m2_aggregates_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant, metrics_in: sales} - id: M3 description: a filtered aggregate, in the portable form SUM(CASE WHEN ... THEN x END) (no ELSE, equal to FILTER per expression_language.md) and with the REQUIRED FILTER (WHERE ...) clause @@ -121,9 +124,9 @@ rungs: reference_form_primary: qualified variants: - {id: M3.filtered_agg.qualified, file: m3/m3_filtered_agg_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} - - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant} + - {id: M3.filtered_agg.bare, file: m3/m3_filtered_agg_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: reference-form-variant, metrics_in: sales} - {id: M3.filter_clause.qualified, file: m3/m3_filter_clause_qualified.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: filter-clause-form} - - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form} + - {id: M3.filter_clause.bare, file: m3/m3_filter_clause_bare.ossie.yaml, tag: ANSI_SQL, reference_form: bare, role: filter-clause-form, metrics_in: sales} - id: M4 description: two datasets joined by a relationship, with a model-scoped metric across the join @@ -149,10 +152,10 @@ rungs: description: date and time scalars from the portable core (YEAR, DATE_TRUNC, DATEDIFF part-first) elements_under_test: - kind: field - name: order_year + name: sales.order_year construct: YEAR - kind: field - name: order_month_start + name: sales.order_month_start construct: DATE_TRUNC - kind: metric name: avg_fulfillment_days From 337f7fcfc7707fd0b902263c35a0365907ddd9d1 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Tue, 3 Nov 2026 20:28:16 +0100 Subject: [PATCH 11/17] docs(compliance): point the portability ladder at apache/ossie-converters#24 Issue #576 was transferred to apache/ossie-converters#24 with the converters; the ladder, grader and manifest still cited the old number, which now resolves to nothing. Co-authored-by: Isaac --- compliance/portability/README.md | 9 +++++---- compliance/portability/grader/README.md | 6 +++--- compliance/portability/grader/grade.py | 4 ++-- compliance/portability/grader/test_grade.py | 2 +- compliance/portability/manifest.yaml | 2 +- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compliance/portability/README.md b/compliance/portability/README.md index c8a8f4c99..1b8256276 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -20,10 +20,11 @@ # Portability measurement ladder (M0-M6) A small set of portable fixtures, and a grader, for measuring how faithfully a -model survives a round-trip through each converter. This is issue #576 -(measurements for model portability across converters). The fixtures are the -inputs; `grader/` scores each returned document against its input, element by -element, into the outcome vocabulary in `manifest.yaml`. A thin runner that drives +model survives a round-trip through each converter. This is +apache/ossie-converters#24 (measurements for model portability across +converters; formerly #576 here). The fixtures are the inputs; `grader/` scores +each returned document against its input, element by element, into the outcome +vocabulary in `manifest.yaml`. A thin runner that drives the converters (modelled on `cube/tools/interop_matrix.py` in apache/ossie-converters) turns the whole thing into one command; the grader is written to sit behind it. diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index 020bb87b7..43c633fb5 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -19,9 +19,9 @@ # Portability fidelity grader -`grade.py` is the scoring half of issue #576. Conformance (`validate.py` passing) -is a floor and says nothing about whether a model survived a converter round-trip; -this grader is the fidelity half. +`grade.py` is the scoring half of apache/ossie-converters#24 (formerly #576). +Conformance (`validate.py` passing) is a floor and says nothing about whether a +model survived a converter round-trip; this grader is the fidelity half. It does not drive the converters. It takes the input document and the document a converter returned, and diffs them element by element into the outcome vocabulary diff --git a/compliance/portability/grader/grade.py b/compliance/portability/grader/grade.py index 800b93080..b3b31412d 100644 --- a/compliance/portability/grader/grade.py +++ b/compliance/portability/grader/grade.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Fidelity grader for the portability measurement ladder (issue #576). +"""Fidelity grader for the portability measurement ladder (apache/ossie-converters#24, formerly #576). The grader answers one question per element: when a model is driven through a converter and back, what happened to each metric and field expression it started @@ -408,7 +408,7 @@ def _format(report): def main(argv=None): - parser = argparse.ArgumentParser(description="Grade a converter round-trip for #576.") + parser = argparse.ArgumentParser(description="Grade a converter round-trip (apache/ossie-converters#24).") parser.add_argument("--input", required=True, help="the input .ossie.yaml/.json document") parser.add_argument("--returned", required=True, help="the document the converter returned") parser.add_argument("--diagnostics", help="a text file of the converter's warnings/notices") diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 5e1a7b481..2b8d47507 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -"""Outcome tests for the #576 fidelity grader. +"""Outcome tests for the portability fidelity grader (apache/ossie-converters#24). Each test builds a tiny input document and a synthetic returned document that exercises exactly one outcome. The identity test runs the real posted M0-M6 fixtures diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index dd37d3b1c..8b7810a96 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Portability measurement ladder, rungs M0-M3 (strawman for #576). +# Portability measurement ladder, rungs M0-M3 (strawman for apache/ossie-converters#24, formerly #576). # # One capability is added per rung so that a difference in a converter's # round-trip localizes to a single construct. Every rung is authored in a From e001de403f9b7f327485caac3df022873104637c Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Sun, 8 Nov 2026 07:33:44 +0100 Subject: [PATCH 12/17] fix(compliance): keep the dataset qualifier on a field two datasets declare MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit M4 is the first rung with two datasets, and customer_id is declared by both sales and customers. The grader stripped every table qualifier before comparing trees, so a converter that re-pointed COUNT(DISTINCT customers.customer_id) to sales.customer_id, or dropped the qualifiers, was graded as a cosmetic rewrite, and _repair_by_expression paired such a renamed metric as expression-equivalent. _ambiguous_fields() reads the field names that more than one input dataset declares; for those the qualifier is kept, for all others it is stripped as before (M0-M3 have one dataset each, so their grades do not change). Three regression tests; two fail on the previous grader. Suggested-by: Jean-Baptiste Onofré Co-authored-by: Isaac --- compliance/portability/grader/grade.py | 47 ++++++++++++++------ compliance/portability/grader/test_grade.py | 48 +++++++++++++++++++++ 2 files changed, 82 insertions(+), 13 deletions(-) diff --git a/compliance/portability/grader/grade.py b/compliance/portability/grader/grade.py index b3b31412d..a4356fb7d 100644 --- a/compliance/portability/grader/grade.py +++ b/compliance/portability/grader/grade.py @@ -103,11 +103,29 @@ def _normalize(expression): return re.sub(r"\s*([(),])\s*", r"\1", collapsed) -def _equivalent_tree(left, right): +def _ambiguous_fields(document): + """Field names declared by more than one dataset. For these the dataset + qualifier says which column is meant, so normalization must keep it.""" + owners = {} + if isinstance(document, dict): + for dataset in document.get("datasets") or []: + if not isinstance(dataset, dict): + continue + for field_def in dataset.get("fields") or []: + if isinstance(field_def, dict) and field_def.get("name"): + owners.setdefault(str(field_def["name"]).lower(), set()).add(dataset.get("name")) + return frozenset(name for name, datasets in owners.items() if len(datasets) > 1) + + +def _equivalent_tree(left, right, keep_qualified=frozenset()): """Return True if two SQL expressions are equal after stripping table qualifiers and quoting, False if they differ, or None if it cannot be decided (sqlglot missing, or either side does not parse). Used only to tell a cosmetic - REWRITTEN from one that needs review.""" + REWRITTEN from one that needs review. + + keep_qualified holds the field names that more than one dataset declares + (_ambiguous_fields): for those the table qualifier is meaning, not spelling, + so it is kept (sales.customer_id and customers.customer_id differ).""" try: import sqlglot from sqlglot import exp @@ -120,8 +138,9 @@ def canonical(text): except Exception: return None for column in tree.find_all(exp.Column): - # Drop the table qualifier so sales.amount and amount compare equal. - column.set("table", None) + if column.name.lower() not in keep_qualified: + # Drop the table qualifier so sales.amount and amount compare equal. + column.set("table", None) column.set("db", None) column.set("catalog", None) for identifier in tree.find_all(exp.Identifier): @@ -238,9 +257,9 @@ def _diagnosed(name, diagnostics): return name in diagnostics or name.split(".")[-1] in diagnostics -def _rewritten(input_element, authored_text, returned_text): +def _rewritten(input_element, authored_text, returned_text, keep_qualified=frozenset()): """A REWRITTEN finding, with the tree comparison that tells cosmetic from review.""" - equivalent = _equivalent_tree(authored_text, returned_text) + equivalent = _equivalent_tree(authored_text, returned_text, keep_qualified) if equivalent is True: note = "cosmetic (normalized trees equal)" elif equivalent is False: @@ -253,7 +272,7 @@ def _rewritten(input_element, authored_text, returned_text): ) -def _classify_present(input_element, output_element): +def _classify_present(input_element, output_element, keep_qualified=frozenset()): """Classify an input element that still has a counterpart in the output.""" in_dialects = input_element.dialects out_dialects = output_element.dialects @@ -279,7 +298,7 @@ def _classify_present(input_element, output_element): # The changed form is the rewrite to report, ahead of a copy that came back. authored_tag = changed[0] authored_text = in_dialects[authored_tag] - return _rewritten(input_element, authored_text, out_dialects[authored_tag]) + return _rewritten(input_element, authored_text, out_dialects[authored_tag], keep_qualified) if kept: preserved = "OSSIE_SQL_2026" in kept return Finding( @@ -303,7 +322,7 @@ def _classify_present(input_element, output_element): # REWRITTEN: a portable tag is present but the expression text changed. if portable_out: returned_text = portable_out.get(authored_tag) or next(iter(portable_out.values())) - return _rewritten(input_element, authored_text, returned_text) + return _rewritten(input_element, authored_text, returned_text, keep_qualified) # No portable tag: either a placeholder stand-in or a real vendor-only return. if out_dialects: @@ -320,7 +339,7 @@ def _classify_present(input_element, output_element): return Finding(input_element.kind, input_element.name, SURVIVED_VENDOR_ONLY, detail="no expression returned") -def _repair_by_expression(dropped, added): +def _repair_by_expression(dropped, added, keep_qualified=frozenset()): """Pair a DROPPED input element with an ADDED output element that carries an equivalent expression, so a converter that did not preserve an element's name (orionbelt writes a field's expression as its name) scores as one element whose @@ -337,7 +356,7 @@ def _repair_by_expression(dropped, added): continue out_text = next(iter(out_el.dialects.values()), "") if in_text and (_normalize(out_el.name) == in_text or out_text == in_text - or _equivalent_tree(in_text, out_text) is True): + or _equivalent_tree(in_text, out_text, keep_qualified) is True): pairs.append((in_el, out_el)) used.add(id(out_el)) break @@ -358,19 +377,21 @@ def grade(input_document, output_document, diagnostics_text="", converter=""): return report diagnostics = diagnostics_text or "" + # Qualifiers that carry meaning are read from the input, the authored model. + keep_qualified = _ambiguous_fields(input_document) present = {} dropped = [] for key, element in input_elements.items(): if key in output_elements: - present[key] = _classify_present(element, output_elements[key]) + present[key] = _classify_present(element, output_elements[key], keep_qualified) else: dropped.append(element) added_elements = [el for key, el in output_elements.items() if key not in input_elements] # Recover identity: an unnamed-preserving rename shows up as drop + add. - repaired = _repair_by_expression(dropped, added_elements) + repaired = _repair_by_expression(dropped, added_elements, keep_qualified) repaired_in = {id(a): b for a, b in repaired} repaired_out = {id(b) for _, b in repaired} diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 2b8d47507..791d34b4c 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -393,3 +393,51 @@ def test_a_dataset_scoped_metric_moved_to_the_model_is_not_the_same_element(): assert outcomes[("metric", "sales.total_amount")] == grade.REWRITTEN assert ("metric", "total_amount") not in outcomes + + +# M4: a field name declared by two datasets keeps its qualifier. + + +def _m4_document(metric_expression="SUM(sales.amount) / COUNT(DISTINCT customers.customer_id)"): + def field(name): + return {"name": name, "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}} + + return { + "datasets": [ + {"name": "sales", "fields": [field("amount"), field("customer_id")]}, + {"name": "customers", "fields": [field("customer_id")]}, + ], + "metrics": [_metric("revenue_per_customer", "ANSI_SQL", metric_expression)], + } + + +def _m4_metric_finding(returned_expression): + report = grade.grade(_m4_document(), _m4_document(returned_expression)) + return next(f for f in report.findings if f.name == "revenue_per_customer") + + +def test_a_field_shared_by_two_datasets_keeps_its_qualifier(): + # customer_id is declared by sales and customers, so the dataset is meaning. + for returned in ( + "SUM(sales.amount) / COUNT(DISTINCT sales.customer_id)", + "SUM(amount) / COUNT(DISTINCT customer_id)", + ): + finding = _m4_metric_finding(returned) + assert finding.outcome == grade.REWRITTEN, returned + assert finding.detail.startswith("review"), (returned, finding.detail) + + +def test_respelling_a_shared_field_is_still_cosmetic(): + finding = _m4_metric_finding('SUM("sales"."amount") / COUNT(DISTINCT "customers"."customer_id")') + assert finding.detail.startswith("cosmetic"), finding.detail + # amount belongs to one dataset only, so its qualifier is still spelling. + finding = _m4_metric_finding("SUM(amount) / COUNT(DISTINCT customers.customer_id)") + assert finding.detail.startswith("cosmetic"), finding.detail + + +def test_a_renamed_metric_moved_to_another_dataset_is_not_repaired(): + returned = _m4_document() + returned["metrics"] = [_metric("rpc", "ANSI_SQL", "SUM(sales.amount) / COUNT(DISTINCT sales.customer_id)")] + outcomes = {f.name: f.outcome for f in grade.grade(_m4_document(), returned).findings} + assert outcomes["revenue_per_customer"] == grade.DROPPED_SILENT + assert outcomes["rpc"] == grade.ADDED From ae3efaef82c9dc91a8a3b148b486773eef930efd Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Thu, 12 Nov 2026 17:34:59 +0100 Subject: [PATCH 13/17] test(compliance): move the M3-M6 fixtures to spec version 0.3.0.dev0 main moved the in-development version to 0.3.0.dev0 (#691), and the schema pins it, so the fixtures this PR adds declare it too. Co-authored-by: Isaac --- compliance/portability/m3/m3_filter_clause_bare.ossie.yaml | 2 +- compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml | 2 +- compliance/portability/m4/m4_cross_dataset.ossie.yaml | 2 +- compliance/portability/m5/m5_metric_ref.ossie.yaml | 2 +- compliance/portability/m6/m6_date_scalars.ossie.yaml | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml index 25c34536f..2763f78e1 100644 --- a/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml +++ b/compliance/portability/m3/m3_filter_clause_bare.ossie.yaml @@ -23,7 +23,7 @@ # (datasets[].metrics), where core-spec/spec.md (Metric Scoping) requires the # single-part field name. A bare name in a model-scoped metric is not valid # since #486; the qualified primary keeps the model-scoped form. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m3_sales description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, bare reference form." datasets: diff --git a/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml index 49af6d340..b1fa1be43 100644 --- a/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml +++ b/compliance/portability/m3/m3_filter_clause_qualified.ossie.yaml @@ -19,7 +19,7 @@ # WHEN fixtures, written with the aggregate FILTER (WHERE ...) clause that # expression_language.md makes REQUIRED. Comparing the two forms shows whether a # converter handles the clause itself. Variant: QUALIFIED reference form, carried as primary. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m3_sales description: "M3: conditional aggregation with the FILTER (WHERE ...) clause, qualified reference form." datasets: diff --git a/compliance/portability/m4/m4_cross_dataset.ossie.yaml b/compliance/portability/m4/m4_cross_dataset.ossie.yaml index 768eb7901..82af35afc 100644 --- a/compliance/portability/m4/m4_cross_dataset.ossie.yaml +++ b/compliance/portability/m4/m4_cross_dataset.ossie.yaml @@ -19,7 +19,7 @@ # aggregates across the join. A cross-dataset metric must qualify its columns # by dataset to be unambiguous, so this rung has a single qualified fixture # (no bare variant: a bare column would not say which dataset it belongs to). -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m4_sales description: "M4: two datasets, a relationship, and a cross-dataset metric." datasets: diff --git a/compliance/portability/m5/m5_metric_ref.ossie.yaml b/compliance/portability/m5/m5_metric_ref.ossie.yaml index 030624c33..1ae29680e 100644 --- a/compliance/portability/m5/m5_metric_ref.ossie.yaml +++ b/compliance/portability/m5/m5_metric_ref.ossie.yaml @@ -21,7 +21,7 @@ # is carried with its parts inlined: a ratio of two aggregates over qualified # dataset.field references, NULL when there are no orders. The two base metrics # stay alongside it. Single fixture. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m5_sales description: "M5: a composed metric, with the metrics it is built from inlined." datasets: diff --git a/compliance/portability/m6/m6_date_scalars.ossie.yaml b/compliance/portability/m6/m6_date_scalars.ossie.yaml index 8e160493c..273a7ec94 100644 --- a/compliance/portability/m6/m6_date_scalars.ossie.yaml +++ b/compliance/portability/m6/m6_date_scalars.ossie.yaml @@ -19,7 +19,7 @@ # DATE_TRUNC, DATEDIFF). DATEDIFF uses the spec's part-first argument order # DATEDIFF(part, start, end). The date-part fields resolve inside their own # dataset (sibling columns), and the DATEDIFF metric uses the qualified form. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m6_sales description: "M6: date and time scalar functions from the portable core." datasets: From 282fe83ee747f0df26a24d9ee8a1a8c5d20c3a6c Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 13 Nov 2026 06:24:58 +0100 Subject: [PATCH 14/17] fix(compliance): resolve a dataset qualifier against the dataset that declares the field Review of #631. The shared-field rule kept or dropped a qualifier by the field name alone, so a field requalified with its own dataset needed review when another dataset declared the name too, and a metric that named a dataset not declaring the field was cosmetic. Each column is now resolved to the dataset it names before the trees are compared: a bare name in a field expression or dataset-scoped metric is the element's own dataset, elsewhere the one dataset that declares it, and a written qualifier is kept as written. Co-authored-by: Isaac --- compliance/portability/grader/README.md | 6 ++ compliance/portability/grader/grade.py | 83 +++++++++++++-------- compliance/portability/grader/test_grade.py | 33 ++++++++ 3 files changed, 91 insertions(+), 31 deletions(-) diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index 43c633fb5..c82469610 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -54,6 +54,12 @@ PLACEHOLDER, ADDED, RELABELED, DROPPED_DIAGNOSED, DROPPED_SILENT, REJECTED. marked cosmetic, while one it cannot prove equivalent is flagged for review. Without `sqlglot` the grader still runs and marks every rewrite as needing review. +- **A qualifier is checked against the model.** Before trees are compared, each + column is resolved to the dataset it names: a bare name in a field expression + or dataset-scoped metric is the element's own dataset, elsewhere it is the one + dataset that declares it. A requalify that names the same dataset is cosmetic; + one that names another dataset (`customers.amount` for `sales.amount`) needs + review. - **Textual by default, semantic where it can be.** PORTABLE means the converter returned the same expression it was given (modulo cosmetic whitespace). True semantic equivalence of SQL is undecidable in general, so the grader auto-clears diff --git a/compliance/portability/grader/grade.py b/compliance/portability/grader/grade.py index a4356fb7d..c1873b573 100644 --- a/compliance/portability/grader/grade.py +++ b/compliance/portability/grader/grade.py @@ -103,44 +103,64 @@ def _normalize(expression): return re.sub(r"\s*([(),])\s*", r"\1", collapsed) -def _ambiguous_fields(document): - """Field names declared by more than one dataset. For these the dataset - qualifier says which column is meant, so normalization must keep it.""" +def _field_owners(document): + """{field name: the datasets that declare it}, lower-cased, read from the input + document (the authored model), so a dataset qualifier can be checked against it.""" owners = {} if isinstance(document, dict): for dataset in document.get("datasets") or []: if not isinstance(dataset, dict): continue + dataset_name = str(dataset.get("name") or "").lower() for field_def in dataset.get("fields") or []: if isinstance(field_def, dict) and field_def.get("name"): - owners.setdefault(str(field_def["name"]).lower(), set()).add(dataset.get("name")) - return frozenset(name for name, datasets in owners.items() if len(datasets) > 1) - - -def _equivalent_tree(left, right, keep_qualified=frozenset()): - """Return True if two SQL expressions are equal after stripping table - qualifiers and quoting, False if they differ, or None if it cannot be decided - (sqlglot missing, or either side does not parse). Used only to tell a cosmetic - REWRITTEN from one that needs review. - - keep_qualified holds the field names that more than one dataset declares - (_ambiguous_fields): for those the table qualifier is meaning, not spelling, - so it is kept (sales.customer_id and customers.customer_id differ).""" + owners.setdefault(str(field_def["name"]).lower(), set()).add(dataset_name) + return {name: frozenset(datasets) for name, datasets in owners.items()} + + +def _equivalent_tree(left, right, owners=None, home=None): + """Return True if two SQL expressions are equal after resolving dataset + qualifiers and dropping quoting, False if they differ, or None if it cannot be + decided (sqlglot missing, or either side does not parse). Used only to tell a + cosmetic REWRITTEN from one that needs review. + + A qualifier is meaning, not spelling, so each column is resolved to the dataset + it names before comparing: a bare name in a field expression or dataset-scoped + metric names ``home``, the element's own dataset; elsewhere a bare name resolves + to the one dataset that declares it (``owners``, from _field_owners) and stays + bare when several do. A written qualifier is kept as written, so + ``customers.amount`` differs from ``sales.amount`` even if only sales declares + ``amount``. A name no dataset declares, and every name without ``owners``, is + compared without its qualifier, since there is nothing to check it against.""" try: import sqlglot from sqlglot import exp except ImportError: # pragma: no cover - environment dependent return None + home = (home or "").lower() or None + + def resolved(column): + if owners is None: + return None + if home: + return column.table.lower() if column.table else home + declared_by = owners.get(column.name.lower(), frozenset()) + if not declared_by: + return None # no dataset declares it, so there is nothing to check against + if column.table: + return column.table.lower() + return next(iter(declared_by)) if len(declared_by) == 1 else None + def canonical(text): try: tree = sqlglot.parse_one(text) except Exception: return None for column in tree.find_all(exp.Column): - if column.name.lower() not in keep_qualified: - # Drop the table qualifier so sales.amount and amount compare equal. - column.set("table", None) + dataset = resolved(column) + # sales.amount and amount compare equal when amount resolves to sales. + column.set("table", exp.to_identifier(dataset) if dataset else None) column.set("db", None) column.set("catalog", None) for identifier in tree.find_all(exp.Identifier): @@ -190,6 +210,7 @@ class Element: kind: str # "metric" or "field" name: str dialects: dict + dataset: str = "" # the declaring dataset of a field or dataset-scoped metric def _collect(document): @@ -216,11 +237,11 @@ def _collect(document): for field_def in dataset.get("fields") or []: if isinstance(field_def, dict) and field_def.get("name"): name = f"{dataset_name}.{field_def['name']}" - elements[("field", name)] = Element("field", name, _dialect_map(field_def.get("expression"))) + elements[("field", name)] = Element("field", name, _dialect_map(field_def.get("expression")), dataset_name) for metric in dataset.get("metrics") or []: if isinstance(metric, dict) and metric.get("name"): name = f"{dataset_name}.{metric['name']}" - elements[("metric", name)] = Element("metric", name, _dialect_map(metric.get("expression"))) + elements[("metric", name)] = Element("metric", name, _dialect_map(metric.get("expression")), dataset_name) return elements @@ -257,9 +278,9 @@ def _diagnosed(name, diagnostics): return name in diagnostics or name.split(".")[-1] in diagnostics -def _rewritten(input_element, authored_text, returned_text, keep_qualified=frozenset()): +def _rewritten(input_element, authored_text, returned_text, owners=None): """A REWRITTEN finding, with the tree comparison that tells cosmetic from review.""" - equivalent = _equivalent_tree(authored_text, returned_text, keep_qualified) + equivalent = _equivalent_tree(authored_text, returned_text, owners, input_element.dataset) if equivalent is True: note = "cosmetic (normalized trees equal)" elif equivalent is False: @@ -272,7 +293,7 @@ def _rewritten(input_element, authored_text, returned_text, keep_qualified=froze ) -def _classify_present(input_element, output_element, keep_qualified=frozenset()): +def _classify_present(input_element, output_element, owners=None): """Classify an input element that still has a counterpart in the output.""" in_dialects = input_element.dialects out_dialects = output_element.dialects @@ -298,7 +319,7 @@ def _classify_present(input_element, output_element, keep_qualified=frozenset()) # The changed form is the rewrite to report, ahead of a copy that came back. authored_tag = changed[0] authored_text = in_dialects[authored_tag] - return _rewritten(input_element, authored_text, out_dialects[authored_tag], keep_qualified) + return _rewritten(input_element, authored_text, out_dialects[authored_tag], owners) if kept: preserved = "OSSIE_SQL_2026" in kept return Finding( @@ -322,7 +343,7 @@ def _classify_present(input_element, output_element, keep_qualified=frozenset()) # REWRITTEN: a portable tag is present but the expression text changed. if portable_out: returned_text = portable_out.get(authored_tag) or next(iter(portable_out.values())) - return _rewritten(input_element, authored_text, returned_text, keep_qualified) + return _rewritten(input_element, authored_text, returned_text, owners) # No portable tag: either a placeholder stand-in or a real vendor-only return. if out_dialects: @@ -339,7 +360,7 @@ def _classify_present(input_element, output_element, keep_qualified=frozenset()) return Finding(input_element.kind, input_element.name, SURVIVED_VENDOR_ONLY, detail="no expression returned") -def _repair_by_expression(dropped, added, keep_qualified=frozenset()): +def _repair_by_expression(dropped, added, owners=None): """Pair a DROPPED input element with an ADDED output element that carries an equivalent expression, so a converter that did not preserve an element's name (orionbelt writes a field's expression as its name) scores as one element whose @@ -356,7 +377,7 @@ def _repair_by_expression(dropped, added, keep_qualified=frozenset()): continue out_text = next(iter(out_el.dialects.values()), "") if in_text and (_normalize(out_el.name) == in_text or out_text == in_text - or _equivalent_tree(in_text, out_text, keep_qualified) is True): + or _equivalent_tree(in_text, out_text, owners, in_el.dataset) is True): pairs.append((in_el, out_el)) used.add(id(out_el)) break @@ -378,20 +399,20 @@ def grade(input_document, output_document, diagnostics_text="", converter=""): diagnostics = diagnostics_text or "" # Qualifiers that carry meaning are read from the input, the authored model. - keep_qualified = _ambiguous_fields(input_document) + owners = _field_owners(input_document) present = {} dropped = [] for key, element in input_elements.items(): if key in output_elements: - present[key] = _classify_present(element, output_elements[key], keep_qualified) + present[key] = _classify_present(element, output_elements[key], owners) else: dropped.append(element) added_elements = [el for key, el in output_elements.items() if key not in input_elements] # Recover identity: an unnamed-preserving rename shows up as drop + add. - repaired = _repair_by_expression(dropped, added_elements, keep_qualified) + repaired = _repair_by_expression(dropped, added_elements, owners) repaired_in = {id(a): b for a, b in repaired} repaired_out = {id(b) for _, b in repaired} diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 791d34b4c..2663a1eb3 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -435,6 +435,39 @@ def test_respelling_a_shared_field_is_still_cosmetic(): assert finding.detail.startswith("cosmetic"), finding.detail +def test_a_qualifier_on_a_field_the_dataset_does_not_declare_is_not_cosmetic(): + # Only sales declares amount, so customers.amount names a different column + # (review of #631). + finding = _m4_metric_finding("SUM(customers.amount) / COUNT(DISTINCT customers.customer_id)") + assert finding.outcome == grade.REWRITTEN + assert finding.detail.startswith("review"), finding.detail + + +def _m4_field_finding(field_name, returned_expression): + returned = _m4_document() + for field_def in returned["datasets"][0]["fields"]: + if field_def["name"] == field_name: + field_def["expression"]["dialects"][0]["expression"] = returned_expression + report = grade.grade(_m4_document(), returned) + return next(f for f in report.findings if f.name == f"sales.{field_name}") + + +def test_a_field_qualified_by_its_own_dataset_is_cosmetic(): + # In a field expression a bare name means the field's own dataset, so naming + # that dataset is spelling, whether or not another dataset declares the name too + # (review of #631). + for field_name in ("amount", "customer_id"): + finding = _m4_field_finding(field_name, f"sales.{field_name}") + assert finding.outcome == grade.REWRITTEN, field_name + assert finding.detail.startswith("cosmetic"), (field_name, finding.detail) + + +def test_a_field_qualified_by_another_dataset_needs_review(): + finding = _m4_field_finding("customer_id", "customers.customer_id") + assert finding.outcome == grade.REWRITTEN + assert finding.detail.startswith("review"), finding.detail + + def test_a_renamed_metric_moved_to_another_dataset_is_not_repaired(): returned = _m4_document() returned["metrics"] = [_metric("rpc", "ANSI_SQL", "SUM(sales.amount) / COUNT(DISTINCT sales.customer_id)")] From c7e8e3f68d1ee94d82cb3cf4d29beb99882707a5 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 6 Nov 2026 08:33:12 +0100 Subject: [PATCH 15/17] fix(compliance): read window NULL placement in the grader as the authored dialect means it sqlglot's base dialect sorts NULL as the smallest value, the opposite of the OSSIE_SQL_2026 default (#587). The grader therefore marked RANK() OVER (ORDER BY x DESC NULLS LAST), which changes the order, as a cosmetic rewrite of RANK() OVER (ORDER BY x DESC), and the equivalent DESC NULLS FIRST as needing review. OSSIE_SQL_2026 expressions are now compared with NULL as the largest value; ANSI_SQL ones, whose default is the engine's, only count as cosmetic when equal under both orderings. Co-authored-by: Isaac --- compliance/portability/grader/README.md | 7 +++ compliance/portability/grader/grade.py | 61 ++++++++++++++++----- compliance/portability/grader/test_grade.py | 46 ++++++++++++++++ 3 files changed, 100 insertions(+), 14 deletions(-) diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index c82469610..328c791cc 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -64,6 +64,13 @@ PLACEHOLDER, ADDED, RELABELED, DROPPED_DIAGNOSED, DROPPED_SILENT, REJECTED. returned the same expression it was given (modulo cosmetic whitespace). True semantic equivalence of SQL is undecidable in general, so the grader auto-clears only the changes a normalized tree proves equal and leaves the rest for a human. +- **NULL placement in a window `ORDER BY` follows the authored tag.** sqlglot's + base dialect sorts NULL as the smallest value, the opposite of the + `OSSIE_SQL_2026` default (#587). An `OSSIE_SQL_2026` expression is compared + with NULL as the largest value, so making the default explicit (`DESC NULLS + FIRST`) is cosmetic and flipping it (`DESC NULLS LAST`) needs review. An + `ANSI_SQL` expression leaves the default to the engine, so an omitted and an + explicit placement are only cosmetic when they agree under both orderings. - **Identity beyond name.** A dropped input element and an added output element that carry an equivalent expression are paired as one element whose identity (name) was not preserved, rather than scored as a drop plus an unrelated add. diff --git a/compliance/portability/grader/grade.py b/compliance/portability/grader/grade.py index c1873b573..abeaed7b9 100644 --- a/compliance/portability/grader/grade.py +++ b/compliance/portability/grader/grade.py @@ -58,6 +58,7 @@ from __future__ import annotations import argparse +import functools import json import re import sys @@ -118,7 +119,32 @@ def _field_owners(document): return {name: frozenset(datasets) for name, datasets in owners.items()} -def _equivalent_tree(left, right, owners=None, home=None): +@functools.lru_cache(maxsize=None) +def _nulls_large_reader(): + """sqlglot's base dialect, except that NULL sorts as the largest value.""" + from sqlglot.dialects.dialect import Dialect + + class _NullsLarge(Dialect): + NULL_ORDERING = "nulls_are_large" + + return _NullsLarge + + +def _null_orderings(dialect): + """The sqlglot readers to compare an expression under, one per NULL placement + the expression may mean when a window ORDER BY item omits NULLS FIRST/LAST. + + OSSIE_SQL_2026 fixes the default (expression_language.md, #587): NULL sorts as + the largest value, so one reader with that ordering. Any other dialect keeps + its own default, which the expression does not say, so an omitted and an + explicit placement are only proven equal if they agree under both orderings. + """ + if dialect == "OSSIE_SQL_2026": + return (_nulls_large_reader(),) + return (None, _nulls_large_reader()) # None is sqlglot's base dialect, NULLs small + + +def _equivalent_tree(left, right, owners=None, home=None, dialect=None): """Return True if two SQL expressions are equal after resolving dataset qualifiers and dropping quoting, False if they differ, or None if it cannot be decided (sqlglot missing, or either side does not parse). Used only to tell a @@ -131,7 +157,10 @@ def _equivalent_tree(left, right, owners=None, home=None): bare when several do. A written qualifier is kept as written, so ``customers.amount`` differs from ``sales.amount`` even if only sales declares ``amount``. A name no dataset declares, and every name without ``owners``, is - compared without its qualifier, since there is nothing to check it against.""" + compared without its qualifier, since there is nothing to check it against. + + ``dialect`` is the authored tag: it decides what an omitted NULLS FIRST/LAST in + a window ORDER BY means.""" try: import sqlglot from sqlglot import exp @@ -152,9 +181,9 @@ def resolved(column): return column.table.lower() return next(iter(declared_by)) if len(declared_by) == 1 else None - def canonical(text): + def canonical(text, reader): try: - tree = sqlglot.parse_one(text) + tree = sqlglot.parse_one(text, read=reader) except Exception: return None for column in tree.find_all(exp.Column): @@ -175,11 +204,14 @@ def canonical(text): pass return tree.sql(normalize=True) - left_sql = canonical(left) - right_sql = canonical(right) - if left_sql is None or right_sql is None: - return None - return left_sql == right_sql + equal = True + for reader in _null_orderings(dialect): + left_sql = canonical(left, reader) + right_sql = canonical(right, reader) + if left_sql is None or right_sql is None: + return None + equal = equal and left_sql == right_sql + return equal def _dialect_map(expression_obj): @@ -278,9 +310,9 @@ def _diagnosed(name, diagnostics): return name in diagnostics or name.split(".")[-1] in diagnostics -def _rewritten(input_element, authored_text, returned_text, owners=None): +def _rewritten(input_element, authored_text, returned_text, owners=None, dialect=None): """A REWRITTEN finding, with the tree comparison that tells cosmetic from review.""" - equivalent = _equivalent_tree(authored_text, returned_text, owners, input_element.dataset) + equivalent = _equivalent_tree(authored_text, returned_text, owners, input_element.dataset, dialect) if equivalent is True: note = "cosmetic (normalized trees equal)" elif equivalent is False: @@ -319,7 +351,7 @@ def _classify_present(input_element, output_element, owners=None): # The changed form is the rewrite to report, ahead of a copy that came back. authored_tag = changed[0] authored_text = in_dialects[authored_tag] - return _rewritten(input_element, authored_text, out_dialects[authored_tag], owners) + return _rewritten(input_element, authored_text, out_dialects[authored_tag], owners, authored_tag) if kept: preserved = "OSSIE_SQL_2026" in kept return Finding( @@ -343,7 +375,7 @@ def _classify_present(input_element, output_element, owners=None): # REWRITTEN: a portable tag is present but the expression text changed. if portable_out: returned_text = portable_out.get(authored_tag) or next(iter(portable_out.values())) - return _rewritten(input_element, authored_text, returned_text, owners) + return _rewritten(input_element, authored_text, returned_text, owners, authored_tag) # No portable tag: either a placeholder stand-in or a real vendor-only return. if out_dialects: @@ -371,13 +403,14 @@ def _repair_by_expression(dropped, added, owners=None): pairs = [] used = set() for in_el in dropped: + in_tag = next(iter(in_el.dialects), None) in_text = next(iter(in_el.dialects.values()), "") for out_el in added: if id(out_el) in used: continue out_text = next(iter(out_el.dialects.values()), "") if in_text and (_normalize(out_el.name) == in_text or out_text == in_text - or _equivalent_tree(in_text, out_text, owners, in_el.dataset) is True): + or _equivalent_tree(in_text, out_text, owners, in_el.dataset, in_tag) is True): pairs.append((in_el, out_el)) used.add(id(out_el)) break diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 2663a1eb3..219462994 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -474,3 +474,49 @@ def test_a_renamed_metric_moved_to_another_dataset_is_not_repaired(): outcomes = {f.name: f.outcome for f in grade.grade(_m4_document(), returned).findings} assert outcomes["revenue_per_customer"] == grade.DROPPED_SILENT assert outcomes["rpc"] == grade.ADDED + + +# ── Window ORDER BY NULL placement (#587) ────────────────────────────────────── + + +def _window_field_finding(dialect, authored, returned): + src = _doc(datasets=[_field("sales", "f", dialect, authored)]) + out = _doc(datasets=[_field("sales", "f", dialect, returned)]) + return _only(grade.grade(src, out)) + + +def test_ossie_sql_default_null_placement_made_explicit_is_cosmetic(): + # OSSIE_SQL_2026 puts NULLs first for DESC and last for ASC when the clause is omitted. + if grade._equivalent_tree("1", "1") is None: + return # sqlglot not installed: every rewrite is marked for review + for authored, returned in [ + ("RANK() OVER (ORDER BY amount DESC)", "RANK() OVER (ORDER BY amount DESC NULLS FIRST)"), + ("LAG(amount) OVER (ORDER BY order_date)", "LAG(amount) OVER (ORDER BY order_date NULLS LAST)"), + ]: + finding = _window_field_finding("OSSIE_SQL_2026", authored, returned) + assert finding.outcome == grade.REWRITTEN + assert "cosmetic" in finding.detail, (returned, finding.detail) + + +def test_ossie_sql_flipped_null_placement_needs_review(): + if grade._equivalent_tree("1", "1") is None: + return + for authored, returned in [ + ("RANK() OVER (ORDER BY amount DESC)", "RANK() OVER (ORDER BY amount DESC NULLS LAST)"), + ("LAG(amount) OVER (ORDER BY order_date)", "LAG(amount) OVER (ORDER BY order_date NULLS FIRST)"), + ]: + finding = _window_field_finding("OSSIE_SQL_2026", authored, returned) + assert finding.outcome == grade.REWRITTEN + assert "review" in finding.detail, (returned, finding.detail) + + +def test_ansi_sql_omitted_null_placement_is_not_proven_equal_to_an_explicit_one(): + # ANSI_SQL leaves the default to the engine, so neither explicit placement is proven equal. + if grade._equivalent_tree("1", "1") is None: + return + for returned in [ + "LAG(amount) OVER (ORDER BY order_date NULLS FIRST)", + "LAG(amount) OVER (ORDER BY order_date NULLS LAST)", + ]: + finding = _window_field_finding("ANSI_SQL", "LAG(amount) OVER (ORDER BY order_date)", returned) + assert "review" in finding.detail, (returned, finding.detail) From d0b1bb748451e42c65fdd533e77f3c799da151a8 Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 6 Nov 2026 08:34:13 +0100 Subject: [PATCH 16/17] test(compliance): add portability rungs M7 (window ORDER BY) and M8 (ratio-to-total) M7 covers window functions with an ORDER BY: LAG, a running SUM with a ROWS frame, and RANK() OVER (ORDER BY ... DESC) without NULLS FIRST/LAST, which relies on the OSSIE_SQL_2026 default from #587. That default is defined for OSSIE_SQL_2026 only, so M7 is authored in it. M8 covers ratio-to-total, x / SUM(x) OVER () and OVER (PARTITION BY ...), with no ORDER BY. Both are field expressions over sibling columns, as M1. The manifest, the identity test and both READMEs list M0-M8; the hold note on #587 is gone. All 15 fixtures validate on main (f28692a) with sqlglot 30.21.0. Co-authored-by: Isaac --- compliance/portability/README.md | 8 +-- compliance/portability/grader/README.md | 2 +- compliance/portability/grader/test_grade.py | 8 +-- .../portability/m7/m7_window_order.ossie.yaml | 61 +++++++++++++++++++ .../m8/m8_ratio_to_total.ossie.yaml | 52 ++++++++++++++++ compliance/portability/manifest.yaml | 36 +++++++++-- 6 files changed, 153 insertions(+), 14 deletions(-) create mode 100644 compliance/portability/m7/m7_window_order.ossie.yaml create mode 100644 compliance/portability/m8/m8_ratio_to_total.ossie.yaml diff --git a/compliance/portability/README.md b/compliance/portability/README.md index 1b8256276..6a5619d3d 100644 --- a/compliance/portability/README.md +++ b/compliance/portability/README.md @@ -17,7 +17,7 @@ under the License. --> -# Portability measurement ladder (M0-M6) +# Portability measurement ladder (M0-M8) A small set of portable fixtures, and a grader, for measuring how faithfully a model survives a round-trip through each converter. This is @@ -42,6 +42,8 @@ difference in a converter's output localizes to a single construct: | M4 | a cross-dataset metric | two datasets, a relationship, a model-scoped metric across the join | | M5 | metric composition | a model-scoped metric built from two others, inlined (a metric expression cannot refer to other metrics) | | M6 | date and time scalars | `YEAR`, `DATE_TRUNC`, `DATEDIFF(part, start, end)` | +| M7 | window functions with an `ORDER BY` | `LAG`, a running `SUM` with a `ROWS` frame, `RANK() OVER (ORDER BY ... DESC)` on the default NULL placement (in `OSSIE_SQL_2026`, the dialect that defines it) | +| M8 | ratio-to-total | `x / SUM(x) OVER ()` and `x / SUM(x) OVER (PARTITION BY ...)` | `manifest.yaml` lists the fixtures, the element each rung exercises, and the per-element outcome vocabulary the grader assigns (PORTABLE / TAG_PRESERVED, @@ -65,10 +67,6 @@ grades the real fixtures all-PORTABLE. See `grader/README.md`. ## Scope and placement -Rungs M7 and M8 (window and offset functions: time intelligence and -ratio-to-total) are held until the default NULL placement in window `ORDER BY` -is settled (#587), and are not in this set. - The fixtures measure the specification rather than any one converter, so they live in this repository; the fixtures are self-contained and the manifest paths are relative. diff --git a/compliance/portability/grader/README.md b/compliance/portability/grader/README.md index 328c791cc..1bccbe5f5 100644 --- a/compliance/portability/grader/README.md +++ b/compliance/portability/grader/README.md @@ -80,7 +80,7 @@ PLACEHOLDER, ADDED, RELABELED, DROPPED_DIAGNOSED, DROPPED_SILENT, REJECTED. ## Tests `test_grade.py` has one test per outcome plus an identity check that runs the real -M0-M6 fixtures through a no-op converter and asserts every element grades +M0-M8 fixtures through a no-op converter and asserts every element grades PORTABLE. Two manifest checks assert that every element the manifest names is found by kind and name in each variant, and that each primary variant uses the rung's primary reference form. Run with `PYTHONPATH=. pytest test_grade.py`. diff --git a/compliance/portability/grader/test_grade.py b/compliance/portability/grader/test_grade.py index 219462994..e0c157315 100644 --- a/compliance/portability/grader/test_grade.py +++ b/compliance/portability/grader/test_grade.py @@ -18,7 +18,7 @@ """Outcome tests for the portability fidelity grader (apache/ossie-converters#24). Each test builds a tiny input document and a synthetic returned document that -exercises exactly one outcome. The identity test runs the real posted M0-M6 fixtures +exercises exactly one outcome. The identity test runs the real posted M0-M8 fixtures through a no-op converter: returned unchanged, every element must grade PORTABLE. The manifest tests check that every element the manifest names can be found by kind and name in each variant, and that each primary variant uses the rung's @@ -297,13 +297,13 @@ def test_whitespace_is_not_a_difference(): assert _only(grade.grade(src, returned)).outcome == grade.PORTABLE -def test_identity_on_real_m0_m6_fixtures_is_all_portable(): +def test_identity_on_real_m0_m8_fixtures_is_all_portable(): files = sorted( path - for rung in ("m0", "m1", "m2", "m3", "m4", "m5", "m6") + for rung in ("m0", "m1", "m2", "m3", "m4", "m5", "m6", "m7", "m8") for path in glob.glob(os.path.join(HERE, rung, "*.ossie.yaml")) ) - assert files, "no M0-M6 fixtures found" + assert files, "no M0-M8 fixtures found" for path in files: document = grade._load(path) report = grade.grade(document, document) diff --git a/compliance/portability/m7/m7_window_order.ossie.yaml b/compliance/portability/m7/m7_window_order.ossie.yaml new file mode 100644 index 000000000..8e17036d4 --- /dev/null +++ b/compliance/portability/m7/m7_window_order.ossie.yaml @@ -0,0 +1,61 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# M7 ladder rung: window functions with an ORDER BY, from the portable core +# (offset, windowed aggregate, ranking). Time intelligence the portable way: a +# prior-period value via LAG and a running total via a windowed SUM with an +# explicit frame. The rank orders DESC without NULLS FIRST or NULLS LAST, so it +# relies on the default placement expression_language.md sets for OSSIE_SQL_2026 +# (#587: NULL sorts as the largest value, first for DESC). That default applies +# to OSSIE_SQL_2026 only, so this rung is authored in it. The fields are window +# expressions over sibling columns of their own dataset. +version: "0.2.0.dev0" +name: m7_sales +description: "M7: window functions with an ORDER BY (LAG, running SUM, RANK)." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: OSSIE_SQL_2026 + expression: amount + datatype: Decimal + - name: order_date + expression: + dialects: + - dialect: OSSIE_SQL_2026 + expression: order_date + datatype: Date + - name: prior_day_amount + expression: + dialects: + - dialect: OSSIE_SQL_2026 + expression: LAG(amount) OVER (ORDER BY order_date) + datatype: Decimal + - name: running_total_amount + expression: + dialects: + - dialect: OSSIE_SQL_2026 + expression: SUM(amount) OVER (ORDER BY order_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + datatype: Decimal + - name: amount_rank + expression: + dialects: + - dialect: OSSIE_SQL_2026 + expression: RANK() OVER (ORDER BY amount DESC) + datatype: Integer diff --git a/compliance/portability/m8/m8_ratio_to_total.ossie.yaml b/compliance/portability/m8/m8_ratio_to_total.ossie.yaml new file mode 100644 index 000000000..33bc8f48f --- /dev/null +++ b/compliance/portability/m8/m8_ratio_to_total.ossie.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# M8 ladder rung: ratio-to-total with a windowed aggregate and no ORDER BY, so +# NULL placement does not enter. Each row's amount over the all-rows total +# (OVER ()) and over its region's total (OVER (PARTITION BY region)), the +# "Percent of total" form in expression_language.md. The fields are window +# expressions over sibling columns of their own dataset. +version: "0.2.0.dev0" +name: m8_sales +description: "M8: ratio-to-total via a windowed aggregate (OVER (), PARTITION BY)." +datasets: + - name: sales + source: samples.sales.orders + fields: + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + datatype: Decimal + - name: region + expression: + dialects: + - dialect: ANSI_SQL + expression: region + datatype: String + - name: pct_of_total + expression: + dialects: + - dialect: ANSI_SQL + expression: amount / SUM(amount) OVER () + datatype: Decimal + - name: pct_of_region + expression: + dialects: + - dialect: ANSI_SQL + expression: amount / SUM(amount) OVER (PARTITION BY region) + datatype: Decimal diff --git a/compliance/portability/manifest.yaml b/compliance/portability/manifest.yaml index 8b7810a96..7bbc09114 100644 --- a/compliance/portability/manifest.yaml +++ b/compliance/portability/manifest.yaml @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -# Portability measurement ladder, rungs M0-M3 (strawman for apache/ossie-converters#24, formerly #576). +# Portability measurement ladder, rungs M0-M8 (strawman for apache/ossie-converters#24, formerly #576). # # One capability is added per rung so that a difference in a converter's # round-trip localizes to a single construct. Every rung is authored in a @@ -41,9 +41,8 @@ # dataset.metric. M1's element is an intra-dataset field expression, so it # carries a single fixture. # -# All fixtures validate.py-pass on apache/ossie main (9531429) with sqlglot -# 29.0.1. Rungs M7 and M8 (window and offset functions) are held pending #587; -# they are not in this set. +# All fixtures validate.py-pass on apache/ossie main (f28692a) with sqlglot +# 30.21.0. outcomes: # per-element, assigned by the grader after diffing input vs returned output @@ -163,3 +162,32 @@ rungs: reference_form_primary: qualified # the metric qualifies dataset.field; the fields reference sibling columns bare (intra-dataset) variants: - {id: M6.date_scalars, file: m6/m6_date_scalars.ossie.yaml, tag: ANSI_SQL, reference_form: qualified, role: primary} + + - id: M7 + description: window functions with an ORDER BY (LAG, running SUM with a frame, RANK relying on the default NULL placement) + elements_under_test: + - kind: field + name: sales.prior_day_amount + construct: LAG(x) OVER (ORDER BY ...) + - kind: field + name: sales.running_total_amount + construct: windowed SUM with ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + - kind: field + name: sales.amount_rank + construct: RANK() OVER (ORDER BY ... DESC) without NULLS FIRST/LAST, the OSSIE_SQL_2026 default (#587) + reference_form_primary: n/a # window field expressions reference sibling columns bare (intra-dataset) + variants: + - {id: M7.window_order, file: m7/m7_window_order.ossie.yaml, tag: OSSIE_SQL_2026, reference_form: intra-dataset, role: primary} + + - id: M8 + description: ratio-to-total with a windowed aggregate and no ORDER BY (share of the all-rows total and of the per-partition total) + elements_under_test: + - kind: field + name: sales.pct_of_total + construct: x / SUM(x) OVER () + - kind: field + name: sales.pct_of_region + construct: x / SUM(x) OVER (PARTITION BY ...) + reference_form_primary: n/a # window field expressions reference sibling columns bare (intra-dataset) + variants: + - {id: M8.ratio_to_total, file: m8/m8_ratio_to_total.ossie.yaml, tag: ANSI_SQL, reference_form: intra-dataset, role: primary} From fad84ebe825a47cdad92e9570219d09dcec3894c Mon Sep 17 00:00:00 2001 From: "Mr. Black 1995" Date: Fri, 13 Nov 2026 06:41:07 +0100 Subject: [PATCH 17/17] test(compliance): move the M7 and M8 fixtures to spec version 0.3.0.dev0 Co-authored-by: Isaac --- compliance/portability/m7/m7_window_order.ossie.yaml | 2 +- compliance/portability/m8/m8_ratio_to_total.ossie.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compliance/portability/m7/m7_window_order.ossie.yaml b/compliance/portability/m7/m7_window_order.ossie.yaml index 8e17036d4..6daf72f71 100644 --- a/compliance/portability/m7/m7_window_order.ossie.yaml +++ b/compliance/portability/m7/m7_window_order.ossie.yaml @@ -22,7 +22,7 @@ # (#587: NULL sorts as the largest value, first for DESC). That default applies # to OSSIE_SQL_2026 only, so this rung is authored in it. The fields are window # expressions over sibling columns of their own dataset. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m7_sales description: "M7: window functions with an ORDER BY (LAG, running SUM, RANK)." datasets: diff --git a/compliance/portability/m8/m8_ratio_to_total.ossie.yaml b/compliance/portability/m8/m8_ratio_to_total.ossie.yaml index 33bc8f48f..160b402ce 100644 --- a/compliance/portability/m8/m8_ratio_to_total.ossie.yaml +++ b/compliance/portability/m8/m8_ratio_to_total.ossie.yaml @@ -19,7 +19,7 @@ # (OVER ()) and over its region's total (OVER (PARTITION BY region)), the # "Percent of total" form in expression_language.md. The fields are window # expressions over sibling columns of their own dataset. -version: "0.2.0.dev0" +version: "0.3.0.dev0" name: m8_sales description: "M8: ratio-to-total via a windowed aggregate (OVER (), PARTITION BY)." datasets: