From 1c473bc2de1626f03007213226abd38e68b50cfb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 4 Jul 2026 17:28:22 +0000 Subject: [PATCH 01/49] =?UTF-8?q?feat:=20holdout=20support=20=E2=80=94=20n?= =?UTF-8?q?ormalized=20models,=20assignment=20precedence,=20exposure=20fie?= =?UTF-8?q?lds,=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Squashed restore of feat/holdouts work (original commits lost to a tooling accident; content fully preserved): - ExperimentHoldout json model; ContextData.holdouts[] + Experiment.holdoutIds[] (normalized payload: definitions not duplicated per experiment) - Context: holdout check after override, before audience/traffic/variant (Override > Holdout > Audience > Traffic > Variant); id->def map with graceful skip of unknown ids; cache invalidation on holdoutIds and resolved definition changes - Exposure heldOut + holdoutId, serializer round-trip - ContextHoldoutTest matrix + serializer/deserializer coverage --- .../main/java/com/absmartly/sdk/Context.java | 145 +++++-- .../com/absmartly/sdk/json/ContextData.java | 14 +- .../com/absmartly/sdk/json/Experiment.java | 7 +- .../absmartly/sdk/json/ExperimentHoldout.java | 64 +++ .../java/com/absmartly/sdk/json/Exposure.java | 14 +- .../com/absmartly/sdk/ContextHoldoutTest.java | 385 ++++++++++++++++++ .../java/com/absmartly/sdk/ContextTest.java | 44 +- .../DefaultContextDataDeserializerTest.java | 40 ++ .../DefaultContextEventSerializerTest.java | 5 +- .../absmartly/sdk/json/ContextDataTest.java | 44 ++ .../sdk/json/ExperimentHoldoutTest.java | 31 ++ .../src/test/resources/holdouts_context.json | 63 +++ 12 files changed, 785 insertions(+), 71 deletions(-) create mode 100644 core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java create mode 100644 core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java create mode 100644 core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java create mode 100644 core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java create mode 100644 core-api/src/test/resources/holdouts_context.json diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index a3d99b3..c22a90c 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -392,6 +392,8 @@ private void queueExposure(final Assignment assignment) { exposure.fullOn = assignment.fullOn; exposure.custom = assignment.custom; exposure.audienceMismatch = assignment.audienceMismatch; + exposure.heldOut = assignment.heldOut; + exposure.holdoutId = assignment.holdoutId; try { eventLock_.lock(); @@ -692,12 +694,14 @@ private void checkReady(final boolean expectNotClosed) { } } - private boolean experimentMatches(final Experiment experiment, final Assignment assignment) { - return experiment.id == assignment.id && - experiment.unitType.equals(assignment.unitType) && - experiment.iteration == assignment.iteration && - experiment.fullOnVariant == assignment.fullOnVariant && - Arrays.equals(experiment.trafficSplit, assignment.trafficSplit); + private boolean experimentMatches(final ContextExperiment experiment, final Assignment assignment) { + return experiment.data.id == assignment.id && + experiment.data.unitType.equals(assignment.unitType) && + experiment.data.iteration == assignment.iteration && + experiment.data.fullOnVariant == assignment.fullOnVariant && + Arrays.equals(experiment.data.trafficSplit, assignment.trafficSplit) && + Arrays.equals(experiment.data.holdoutIds, assignment.holdoutIds) && + Arrays.equals(experiment.holdouts, assignment.holdouts); } private static class Assignment { @@ -715,6 +719,10 @@ private static class Assignment { boolean custom; boolean audienceMismatch; + boolean heldOut; + int holdoutId; + int[] holdoutIds; + ExperimentHoldout[] holdouts; Map variables = Collections.emptyMap(); final AtomicBoolean exposed = new AtomicBoolean(false); @@ -743,7 +751,7 @@ private Assignment getAssignment(final String experimentName) { return assignment; } } else if ((custom == null) || custom == assignment.variant) { - if (experimentMatches(experiment.data, assignment)) { + if (experimentMatches(experiment, assignment)) { // assignment up-to-date return assignment; } @@ -778,50 +786,73 @@ private Assignment getAssignment(final String experimentName) { if (experiment != null) { final String unitType = experiment.data.unitType; - if (experiment.data.audience != null && experiment.data.audience.length() > 0) { - final Map attrs = new HashMap(attributes_.size()); - for (final Attribute attr : attributes_) { - attrs.put(attr.name, attr.value); - } - - final AudienceMatcher.Result match = audienceMatcher_ - .evaluate(experiment.data.audience, attrs); - if (match != null) { - assignment.audienceMismatch = !match.get(); - } - } + assignment.holdoutIds = experiment.data.holdoutIds; + assignment.holdouts = experiment.holdouts; - if (experiment.data.audienceStrict && assignment.audienceMismatch) { - assignment.variant = 0; - } else if (experiment.data.fullOnVariant == 0) { - final String uid = units_.get(experiment.data.unitType); + if (experiment.holdouts != null && experiment.holdouts.length > 0) { + final String uid = units_.get(unitType); if (uid != null) { final byte[] unitHash = Context.this.getUnitHash(unitType, uid); - final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); - final boolean eligible = assigner.assign(experiment.data.trafficSplit, - experiment.data.trafficSeedHi, - experiment.data.trafficSeedLo) == 1; - if (eligible) { - if (custom != null) { - assignment.variant = custom; - assignment.custom = true; + for (final ExperimentHoldout holdout : experiment.holdouts) { + if (assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo) == 0) { + assignment.heldOut = true; + assignment.holdoutId = holdout.id; + assignment.variant = 0; + assignment.assigned = true; + break; + } + } + } + } + + if (!assignment.heldOut) { + if (experiment.data.audience != null && experiment.data.audience.length() > 0) { + final Map attrs = new HashMap(attributes_.size()); + for (final Attribute attr : attributes_) { + attrs.put(attr.name, attr.value); + } + + final AudienceMatcher.Result match = audienceMatcher_ + .evaluate(experiment.data.audience, attrs); + if (match != null) { + assignment.audienceMismatch = !match.get(); + } + } + + if (experiment.data.audienceStrict && assignment.audienceMismatch) { + assignment.variant = 0; + } else if (experiment.data.fullOnVariant == 0) { + final String uid = units_.get(experiment.data.unitType); + if (uid != null) { + final byte[] unitHash = Context.this.getUnitHash(unitType, uid); + + final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, + unitHash); + final boolean eligible = assigner.assign(experiment.data.trafficSplit, + experiment.data.trafficSeedHi, + experiment.data.trafficSeedLo) == 1; + if (eligible) { + if (custom != null) { + assignment.variant = custom; + assignment.custom = true; + } else { + assignment.variant = assigner.assign(experiment.data.split, + experiment.data.seedHi, + experiment.data.seedLo); + } } else { - assignment.variant = assigner.assign(experiment.data.split, - experiment.data.seedHi, - experiment.data.seedLo); + assignment.eligible = false; + assignment.variant = 0; } - } else { - assignment.eligible = false; - assignment.variant = 0; + assignment.assigned = true; } + } else { assignment.assigned = true; + assignment.variant = experiment.data.fullOnVariant; + assignment.fullOn = true; } - } else { - assignment.assigned = true; - assignment.variant = experiment.data.fullOnVariant; - assignment.fullOn = true; } assignment.unitType = unitType; @@ -944,6 +975,7 @@ private void clearRefreshTimer() { private static class ContextExperiment { Experiment data; + ExperimentHoldout[] holdouts; List> variables; Map customFieldValues; } @@ -953,13 +985,44 @@ private static class ContextCustomFieldValue { Object value; } + private static ExperimentHoldout[] resolveHoldouts(final int[] holdoutIds, + final Map holdoutIndex) { + if (holdoutIds == null || holdoutIds.length == 0) { + return null; + } + + final List resolved = new ArrayList(holdoutIds.length); + for (final int holdoutId : holdoutIds) { + final ExperimentHoldout holdout = holdoutIndex.get(holdoutId); + if (holdout != null && holdout.split != null && holdout.split.length > 0) { + resolved.add(holdout); + } + } + + if (resolved.isEmpty()) { + return null; + } + + return resolved.toArray(new ExperimentHoldout[0]); + } + private void setData(final ContextData data) { final Map index = new HashMap(); final Map> indexVariables = new HashMap>(); + final Map holdoutIndex = new HashMap(); + if (data.holdouts != null) { + for (final ExperimentHoldout holdout : data.holdouts) { + if (holdout != null) { + holdoutIndex.put(holdout.id, holdout); + } + } + } + for (final Experiment experiment : data.experiments) { final ContextExperiment contextExperiment = new ContextExperiment(); contextExperiment.data = experiment; + contextExperiment.holdouts = resolveHoldouts(experiment.holdoutIds, holdoutIndex); contextExperiment.variables = new ArrayList>(experiment.variants.length); for (final ExperimentVariant variant : experiment.variants) { diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java index 96139aa..9833f2e 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java @@ -11,6 +11,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ContextData { public Experiment[] experiments = new Experiment[0]; + public ExperimentHoldout[] holdouts = new ExperimentHoldout[0]; public ContextData() {} @@ -19,6 +20,12 @@ public ContextData(Experiment[] experiments) { this.experiments = experiments; } + @SuppressFBWarnings(value = "EI_EXPOSE_REP2") + public ContextData(Experiment[] experiments, ExperimentHoldout[] holdouts) { + this.experiments = experiments; + this.holdouts = holdouts; + } + @Override public boolean equals(Object o) { if (this == o) @@ -26,18 +33,21 @@ public boolean equals(Object o) { if (o == null || getClass() != o.getClass()) return false; ContextData that = (ContextData) o; - return Arrays.equals(experiments, that.experiments); + return Arrays.equals(experiments, that.experiments) && Arrays.equals(holdouts, that.holdouts); } @Override public int hashCode() { - return Arrays.hashCode(experiments); + int result = Arrays.hashCode(experiments); + result = 31 * result + Arrays.hashCode(holdouts); + return result; } @Override public String toString() { return "ContextData{" + "experiments=" + Arrays.toString(experiments) + + ", holdouts=" + Arrays.toString(holdouts) + '}'; } } diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java index 77a3420..e3a8459 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java @@ -24,6 +24,7 @@ public class Experiment { public boolean audienceStrict; public String audience; public CustomFieldValue[] customFieldValues; + public int[] holdoutIds; public Experiment() {} @@ -66,7 +67,9 @@ public boolean equals(Object o) { return false; if (audience != null ? !audience.equals(that.audience) : that.audience != null) return false; - return Arrays.equals(customFieldValues, that.customFieldValues); + if (!Arrays.equals(customFieldValues, that.customFieldValues)) + return false; + return Arrays.equals(holdoutIds, that.holdoutIds); } @Override @@ -87,6 +90,7 @@ public int hashCode() { result = 31 * result + (audienceStrict ? 1 : 0); result = 31 * result + (audience != null ? audience.hashCode() : 0); result = 31 * result + Arrays.hashCode(customFieldValues); + result = 31 * result + Arrays.hashCode(holdoutIds); return result; } @@ -109,6 +113,7 @@ public String toString() { ", audienceStrict=" + audienceStrict + ", audience='" + audience + '\'' + ", customFieldValues=" + Arrays.toString(customFieldValues) + + ", holdoutIds=" + Arrays.toString(holdoutIds) + '}'; } } diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java b/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java new file mode 100644 index 0000000..cbaf4e6 --- /dev/null +++ b/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java @@ -0,0 +1,64 @@ +package com.absmartly.sdk.json; + +import java.util.Arrays; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; + +import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; + +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +public class ExperimentHoldout { + public int id; + public int seedHi; + public int seedLo; + public double[] split; + + public ExperimentHoldout() {} + + @SuppressFBWarnings(value = "EI_EXPOSE_REP2") + public ExperimentHoldout(int id, int seedHi, int seedLo, double[] split) { + this.id = id; + this.seedHi = seedHi; + this.seedLo = seedLo; + this.split = split; + } + + @Override + public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + + ExperimentHoldout that = (ExperimentHoldout) o; + + if (id != that.id) + return false; + if (seedHi != that.seedHi) + return false; + if (seedLo != that.seedLo) + return false; + return Arrays.equals(split, that.split); + } + + @Override + public int hashCode() { + int result = id; + result = 31 * result + seedHi; + result = 31 * result + seedLo; + result = 31 * result + Arrays.hashCode(split); + return result; + } + + @Override + public String toString() { + return "ExperimentHoldout{" + + "id=" + id + + ", seedHi=" + seedHi + + ", seedLo=" + seedLo + + ", split=" + Arrays.toString(split) + + '}'; + } +} diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java b/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java index 5c0e158..6908582 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java @@ -19,11 +19,14 @@ public class Exposure { public boolean fullOn; public boolean custom; public boolean audienceMismatch; + public boolean heldOut; + public int holdoutId; public Exposure() {} public Exposure(int id, String name, String unit, int variant, long exposedAt, boolean assigned, boolean eligible, - boolean overridden, boolean fullOn, boolean custom, boolean audienceMismatch) { + boolean overridden, boolean fullOn, boolean custom, boolean audienceMismatch, boolean heldOut, + int holdoutId) { this.id = id; this.name = name; this.unit = unit; @@ -35,6 +38,8 @@ public Exposure(int id, String name, String unit, int variant, long exposedAt, b this.fullOn = fullOn; this.custom = custom; this.audienceMismatch = audienceMismatch; + this.heldOut = heldOut; + this.holdoutId = holdoutId; } @Override @@ -48,13 +53,14 @@ public boolean equals(Object o) { && assigned == exposure.assigned && eligible == exposure.eligible && overridden == exposure.overridden && fullOn == exposure.fullOn && custom == exposure.custom && Objects.equals(audienceMismatch, exposure.audienceMismatch) && Objects.equals(name, exposure.name) - && Objects.equals(unit, exposure.unit); + && Objects.equals(unit, exposure.unit) && heldOut == exposure.heldOut + && holdoutId == exposure.holdoutId; } @Override public int hashCode() { return Objects.hash(id, name, unit, variant, exposedAt, assigned, eligible, overridden, fullOn, custom, - audienceMismatch); + audienceMismatch, heldOut, holdoutId); } @Override @@ -71,6 +77,8 @@ public String toString() { ", fullOn=" + fullOn + ", custom=" + custom + ", audienceMismatch=" + audienceMismatch + + ", heldOut=" + heldOut + + ", holdoutId=" + holdoutId + '}'; } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java new file mode 100644 index 0000000..3b6e615 --- /dev/null +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -0,0 +1,385 @@ +package com.absmartly.sdk; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.concurrent.ScheduledExecutorService; +import java8.util.concurrent.CompletableFuture; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import com.absmartly.sdk.internal.hashing.Hashing; +import com.absmartly.sdk.java.nio.charset.StandardCharsets; +import com.absmartly.sdk.java.time.Clock; +import com.absmartly.sdk.json.ContextData; +import com.absmartly.sdk.json.Experiment; +import com.absmartly.sdk.json.ExperimentApplication; +import com.absmartly.sdk.json.ExperimentHoldout; +import com.absmartly.sdk.json.ExperimentVariant; +import com.absmartly.sdk.json.Exposure; +import com.absmartly.sdk.json.PublishEvent; +import com.absmartly.sdk.json.Unit; + +class ContextHoldoutTest extends TestUtils { + static final String UNIT_TYPE = "session_id"; + static final String UID = "e791e240fcd3df7d238cfc285f475e8152fcc0ec"; + + // split[0.5,0.5], seedHi=100, seedLo=200 -> variant 1 for UID above. + static final int NORMAL_SEED_HI = 100; + static final int NORMAL_SEED_LO = 200; + static final int NORMAL_VARIANT = 1; + + // split[0.1,0.9], seedHi=13, seedLo=111 -> variant 0 (i.e. user IS in this holdout) for UID above. + static final int HOLDOUT_IN_SEED_HI = 13; + static final int HOLDOUT_IN_SEED_LO = 111; + + // split[0.1,0.9], seedHi=1, seedLo=222 -> variant 1 (i.e. user is NOT in this holdout) for UID above. + static final int HOLDOUT_OUT_SEED_HI = 1; + static final int HOLDOUT_OUT_SEED_LO = 222; + + ContextDataProvider dataProvider; + ContextEventLogger eventLogger; + ContextEventHandler eventHandler; + VariableParser variableParser; + AudienceMatcher audienceMatcher; + ScheduledExecutorService scheduler; + Clock clock = Clock.fixed(1_620_000_000_000L); + + @BeforeEach + void setUp() { + dataProvider = mock(ContextDataProvider.class); + eventHandler = mock(ContextEventHandler.class); + eventLogger = mock(ContextEventLogger.class); + variableParser = new DefaultVariableParser(); + audienceMatcher = new AudienceMatcher(new DefaultAudienceDeserializer()); + scheduler = mock(ScheduledExecutorService.class); + } + + Context createReadyContext(ContextData data) { + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID); + return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, + eventHandler, eventLogger, variableParser, audienceMatcher); + } + + Context createReadyContext(ContextConfig config, ContextData data) { + return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, + eventHandler, eventLogger, variableParser, audienceMatcher); + } + + static Experiment newExperiment(int id, String name) { + final Experiment experiment = new Experiment(); + experiment.id = id; + experiment.name = name; + experiment.unitType = UNIT_TYPE; + experiment.iteration = 1; + experiment.seedHi = NORMAL_SEED_HI; + experiment.seedLo = NORMAL_SEED_LO; + experiment.split = new double[]{0.5, 0.5}; + experiment.trafficSeedHi = 1; + experiment.trafficSeedLo = 2; + experiment.trafficSplit = new double[]{0.0, 1.0}; + experiment.fullOnVariant = 0; + experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + experiment.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + experiment.audienceStrict = false; + experiment.audience = null; + return experiment; + } + + static ExperimentHoldout newHoldout(int id, int seedHi, int seedLo) { + return new ExperimentHoldout(id, seedHi, seedLo, new double[]{0.1, 0.9}); + } + + static ContextData contextDataOf(Experiment... experiments) { + return contextDataOf(new ExperimentHoldout[0], experiments); + } + + static ContextData contextDataOf(ExperimentHoldout[] holdouts, Experiment... experiments) { + final ContextData data = new ContextData(); + data.experiments = experiments; + data.holdouts = holdouts; + return data; + } + + @Test + void assignsNormallyWhenExperimentHasNoHoldouts() { + final Experiment experiment = newExperiment(1, "exp_no_holdout"); + + final Context context = createReadyContext(contextDataOf(experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_holdout")); + } + + @Test + void assignsControlVariantWhenUnitIsInHoldout() { + final Experiment experiment = newExperiment(1, "exp_holdout_in"); + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_holdout_in")); + } + + @Test + void assignsNormallyWhenUnitIsNotInHoldout() { + final Experiment experiment = newExperiment(1, "exp_holdout_out"); + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_holdout_out")); + } + + @Test + void checksAllHoldoutsUntilAMatchIsFound() { + final Experiment experiment = newExperiment(1, "exp_multi_holdout"); + experiment.holdoutIds = new int[]{11, 12}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{ + newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), + newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + }, experiment)); + + assertEquals(0, context.peekTreatment("exp_multi_holdout")); + } + + @Test + void assignsNormallyWhenNotInAnyHoldout() { + final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); + experiment.holdoutIds = new int[]{11, 12}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{ + newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), + newHoldout(12, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), + }, experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_multi_holdout_miss")); + } + + @Test + void sharesHoldoutDefinitionsAcrossExperiments() { + final Experiment experimentA = newExperiment(1, "exp_shared_a"); + experimentA.holdoutIds = new int[]{11}; + final Experiment experimentB = newExperiment(2, "exp_shared_b"); + experimentB.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, + experimentA, experimentB)); + + // same seed -> same membership across both experiments referencing the shared holdout + assertEquals(0, context.peekTreatment("exp_shared_a")); + assertEquals(0, context.peekTreatment("exp_shared_b")); + } + + @Test + void assignsNormallyWhenHoldoutIdIsUnknown() { + final Experiment experiment = newExperiment(1, "exp_unknown_holdout"); + experiment.holdoutIds = new int[]{99}; // no matching top-level definition + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_unknown_holdout")); + } + + @Test + void resolvesKnownHoldoutAndSkipsUnknownId() { + final Experiment experiment = newExperiment(1, "exp_mixed_holdout"); + experiment.holdoutIds = new int[]{99, 11}; // 99 is unknown, 11 resolves and holds the unit out + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_mixed_holdout")); + } + + @Test + void skipsMalformedHoldoutWithNullSplit() { + final Experiment experiment = newExperiment(1, "exp_null_split_holdout"); + experiment.holdoutIds = new int[]{11}; + + // a holdout whose split is missing from the payload must be dropped, not crash assignment + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{new ExperimentHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO, null)}, + experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_null_split_holdout")); + } + + @Test + void skipsHoldoutWhenUnitMissingForUnitType() { + final Experiment experiment = newExperiment(1, "exp_no_unit_holdout"); + experiment.unitType = "user_id"; // no unit registered for this type + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + // holdout evaluation is skipped (no uid); assignment falls through unassigned -> control 0, not held out + assertEquals(0, context.getTreatment("exp_no_unit_holdout")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_no_unit_holdout", "user_id", 0, clock.millis(), false, true, false, false, false, + false, false, 0), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void reusesCachedAssignmentForHeldOutExperiment() { + final Experiment experiment = newExperiment(1, "exp_holdout_cache"); + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cache")); + // second call hits the cache (holdouts present and unchanged) -> no new exposure + assertEquals(0, context.getTreatment("exp_holdout_cache")); + assertEquals(1, context.getPendingCount()); + } + + @Test + void holdoutTakesPrecedenceOverAudienceMismatch() { + final Experiment experiment = newExperiment(1, "exp_holdout_audience"); + experiment.audienceStrict = true; + experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + context.setAttribute("age", 5); // would mismatch the audience filter if evaluated + + assertEquals(0, context.peekTreatment("exp_holdout_audience")); + } + + @Test + void overrideTakesPrecedenceOverHoldout() { + final Experiment experiment = newExperiment(1, "exp_holdout_override"); + experiment.holdoutIds = new int[]{11}; + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setOverride( + "exp_holdout_override", 3); + final Context context = createReadyContext(config, contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(3, context.peekTreatment("exp_holdout_override")); + } + + @Test + void holdoutTakesPrecedenceOverCustomAssignment() { + final Experiment experiment = newExperiment(1, "exp_holdout_custom"); + experiment.holdoutIds = new int[]{11}; + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_holdout_custom", 3); + final Context context = createReadyContext(config, contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_holdout_custom")); + } + + @Test + void refreshReassignsWhenHoldoutsChange() { + final Experiment experiment = newExperiment(1, "exp_holdout_refresh"); + + final Context context = createReadyContext(contextDataOf(experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_refresh")); + assertEquals(1, context.getPendingCount()); + + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_refresh"); + refreshedExperiment.holdoutIds = new int[]{11}; + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_refresh")); + assertEquals(2, context.getPendingCount()); // holdout change triggered a new exposure + } + + @Test + void refreshReassignsWhenReferencedHoldoutDefinitionChanges() { + final Experiment experiment = newExperiment(1, "exp_holdout_def_change"); + experiment.holdoutIds = new int[]{11}; + + // unit is NOT in the holdout initially + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_def_change")); + assertEquals(1, context.getPendingCount()); + + // same holdoutIds, but the referenced definition's seed changes so the unit is now IN the holdout + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_def_change"); + refreshedExperiment.holdoutIds = new int[]{11}; + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_def_change")); + assertEquals(2, context.getPendingCount()); // changed definition triggered a new exposure + } + + @Test + void exposureCarriesHeldOutAndHoldoutId() { + final Experiment experiment = newExperiment(1, "exp_holdout_exposure"); + experiment.holdoutIds = new int[]{42}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(42, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_exposure")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_holdout_exposure", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, + false, true, 42), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } +} diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java index 0abe1c0..d63f473 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java @@ -577,7 +577,7 @@ void setUnitsBeforeReady() { expected.publishedAt = clock.millis(); expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -993,7 +993,7 @@ void getVariableValueQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1021,7 +1021,7 @@ void getVariableValueQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1050,7 +1050,7 @@ void getVariableValueCallsEventLogger() { context.getVariableValue("banner.size", null); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1124,14 +1124,14 @@ void getTreatment() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), new Exposure(2, "exp_test_abc", "session_id", 2, clock.millis(), true, true, false, false, false, - false), + false, false, 0), new Exposure(3, "exp_test_not_eligible", "user_id", 0, clock.millis(), true, false, false, false, - false, false), + false, false, false, 0), new Exposure(4, "exp_test_fullon", "session_id", 2, clock.millis(), true, true, false, true, false, - false), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false), + false, false, 0), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1195,14 +1195,14 @@ void getTreatmentReturnsOverrideVariant() { expected.exposures = new Exposure[]{ new Exposure(1, "exp_test_ab", "session_id", 12, clock.millis(), false, true, true, false, false, - false), + false, false, 0), new Exposure(2, "exp_test_abc", "session_id", 13, clock.millis(), false, true, true, false, false, - false), + false, false, 0), new Exposure(3, "exp_test_not_eligible", "user_id", 11, clock.millis(), false, true, true, false, false, - false), + false, false, 0), new Exposure(4, "exp_test_fullon", "session_id", 13, clock.millis(), false, true, true, false, false, - false), - new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false), + false, false, 0), + new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1266,7 +1266,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1294,7 +1294,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1323,7 +1323,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchTrueAndControlVariantOnAudien expected.exposures = new Exposure[]{ new Exposure(1, "exp_test_ab", "session_id", 0, clock.millis(), false, true, false, false, false, - true), + true, false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1344,8 +1344,8 @@ void getTreatmentCallsEventLogger() { context.getTreatment("not_found"); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, 0), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1553,9 +1553,9 @@ void publishResetsInternalQueuesAndKeepsAttributesOverridesAndCustomAssignments( expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), - new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false), - new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false, false, 0), + new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false, false, 0), }; expected.goals = new GoalAchievement[]{ diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index d1966d8..7471b20 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -7,6 +7,7 @@ import com.absmartly.sdk.json.ContextData; import com.absmartly.sdk.json.Experiment; import com.absmartly.sdk.json.ExperimentApplication; +import com.absmartly.sdk.json.ExperimentHoldout; import com.absmartly.sdk.json.ExperimentVariant; class DefaultContextDataDeserializerTest extends TestUtils { @@ -123,4 +124,43 @@ void deserializeDoesNotThrow() { assertNull(data); }); } + + @Test + void deserializeHoldouts() { + final byte[] bytes = getResourceBytes("holdouts_context.json"); + + final ContextDataDeserializer deser = new DefaultContextDataDeserializer(); + final ContextData data = deser.deserialize(bytes, 0, bytes.length); + + final Experiment experiment = new Experiment(); + experiment.id = 1; + experiment.name = "exp_test_holdout"; + experiment.unitType = "session_id"; + experiment.iteration = 1; + experiment.seedHi = 3603515; + experiment.seedLo = 233373850; + experiment.split = new double[]{0.5, 0.5}; + experiment.trafficSeedHi = 449867249; + experiment.trafficSeedLo = 455443629; + experiment.trafficSplit = new double[]{0.0, 1.0}; + experiment.fullOnVariant = 0; + experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + experiment.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", "{\"banner.border\":1,\"banner.size\":\"large\"}") + }; + experiment.audienceStrict = false; + experiment.audience = null; + experiment.holdoutIds = new int[]{11, 12}; + + final ContextData expected = new ContextData( + new Experiment[]{experiment}, + new ExperimentHoldout[]{ + new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}), + new ExperimentHoldout(12, 1, 222, new double[]{0.05, 0.95}) + }); + + assertNotNull(data); + assertEquals(expected, data); + } } diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java index 5c0dc59..893b307 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java @@ -34,7 +34,8 @@ void serialize() { "nested_arr", mapOf("nested", listOf(1, 2, "test"))); event.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, 123470000L, true, true, false, false, false, true), + new Exposure(1, "exp_test_ab", "session_id", 1, 123470000L, true, true, false, false, false, true, + false, 0), }; event.goals = new GoalAchievement[]{ @@ -54,7 +55,7 @@ void serialize() { final byte[] bytes = ser.serialize(event); assertEquals( - "{\"hashed\":true,\"units\":[{\"type\":\"session_id\",\"uid\":\"pAE3a1i5Drs5mKRNq56adA\"},{\"type\":\"user_id\",\"uid\":\"JfnnlDI7RTiF9RgfG2JNCw\"}],\"publishedAt\":123456789,\"exposures\":[{\"id\":1,\"name\":\"exp_test_ab\",\"unit\":\"session_id\",\"variant\":1,\"exposedAt\":123470000,\"assigned\":true,\"eligible\":true,\"overridden\":false,\"fullOn\":false,\"custom\":false,\"audienceMismatch\":true}],\"goals\":[{\"name\":\"goal1\",\"achievedAt\":123456000,\"properties\":{\"amount\":6,\"nested\":{\"value\":5},\"nested_arr\":{\"nested\":[1,2,\"test\"]},\"tries\":1,\"value\":5.0}},{\"name\":\"goal2\",\"achievedAt\":123456789}],\"attributes\":[{\"name\":\"attr1\",\"value\":\"value1\",\"setAt\":123456000},{\"name\":\"attr2\",\"value\":\"value2\",\"setAt\":123456789},{\"name\":\"attr2\",\"setAt\":123450000},{\"name\":\"attr3\",\"value\":{\"nested\":{\"value\":5}},\"setAt\":123470000},{\"name\":\"attr4\",\"value\":{\"nested\":[1,2,\"test\"]},\"setAt\":123480000}]}", + "{\"hashed\":true,\"units\":[{\"type\":\"session_id\",\"uid\":\"pAE3a1i5Drs5mKRNq56adA\"},{\"type\":\"user_id\",\"uid\":\"JfnnlDI7RTiF9RgfG2JNCw\"}],\"publishedAt\":123456789,\"exposures\":[{\"id\":1,\"name\":\"exp_test_ab\",\"unit\":\"session_id\",\"variant\":1,\"exposedAt\":123470000,\"assigned\":true,\"eligible\":true,\"overridden\":false,\"fullOn\":false,\"custom\":false,\"audienceMismatch\":true,\"heldOut\":false,\"holdoutId\":0}],\"goals\":[{\"name\":\"goal1\",\"achievedAt\":123456000,\"properties\":{\"amount\":6,\"nested\":{\"value\":5},\"nested_arr\":{\"nested\":[1,2,\"test\"]},\"tries\":1,\"value\":5.0}},{\"name\":\"goal2\",\"achievedAt\":123456789}],\"attributes\":[{\"name\":\"attr1\",\"value\":\"value1\",\"setAt\":123456000},{\"name\":\"attr2\",\"value\":\"value2\",\"setAt\":123456789},{\"name\":\"attr2\",\"setAt\":123450000},{\"name\":\"attr3\",\"value\":{\"nested\":{\"value\":5}},\"setAt\":123470000},{\"name\":\"attr4\",\"value\":{\"nested\":[1,2,\"test\"]},\"setAt\":123480000}]}", new String(bytes, StandardCharsets.UTF_8)); } diff --git a/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java new file mode 100644 index 0000000..c798cdc --- /dev/null +++ b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java @@ -0,0 +1,44 @@ +package com.absmartly.sdk.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ContextDataTest { + private static Experiment experiment(int id, String name) { + final Experiment experiment = new Experiment(); + experiment.id = id; + experiment.name = name; + experiment.unitType = "session_id"; + experiment.variants = new ExperimentVariant[0]; + return experiment; + } + + @Test + void equalsHashCodeAndToStringWithHoldouts() { + final Experiment[] experiments = new Experiment[]{experiment(1, "exp")}; + final ExperimentHoldout[] holdouts = new ExperimentHoldout[]{ + new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}) + }; + + final ContextData a = new ContextData(experiments, holdouts); + final ContextData b = new ContextData(experiments, new ExperimentHoldout[]{ + new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}) + }); + + assertEquals(a, a); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + + // same experiments but different holdouts must compare unequal + final ContextData differentHoldouts = new ContextData(experiments, new ExperimentHoldout[]{ + new ExperimentHoldout(22, 13, 111, new double[]{0.1, 0.9}) + }); + assertNotEquals(a, differentHoldouts); + assertNotEquals(a.hashCode(), differentHoldouts.hashCode()); + + assertTrue(a.toString().contains("holdouts=")); + } +} diff --git a/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java new file mode 100644 index 0000000..8176ddd --- /dev/null +++ b/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java @@ -0,0 +1,31 @@ +package com.absmartly.sdk.json; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +class ExperimentHoldoutTest { + @Test + void equalsHashCodeAndToString() { + final ExperimentHoldout a = new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}); + final ExperimentHoldout b = new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}); + + assertEquals(a, a); + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + + assertNotEquals(a, null); + assertNotEquals(a, "not a holdout"); + assertNotEquals(a, new ExperimentHoldout(99, 13, 111, new double[]{0.1, 0.9})); + assertNotEquals(a, new ExperimentHoldout(11, 99, 111, new double[]{0.1, 0.9})); + assertNotEquals(a, new ExperimentHoldout(11, 13, 999, new double[]{0.1, 0.9})); + assertNotEquals(a, new ExperimentHoldout(11, 13, 111, new double[]{0.2, 0.8})); + + final String text = a.toString(); + assertTrue(text.contains("id=11")); + assertTrue(text.contains("seedHi=13")); + assertTrue(text.contains("seedLo=111")); + } +} diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json new file mode 100644 index 0000000..dd64e47 --- /dev/null +++ b/core-api/src/test/resources/holdouts_context.json @@ -0,0 +1,63 @@ +{ + "experiments":[ + { + "id":1, + "name":"exp_test_holdout", + "iteration":1, + "unitType":"session_id", + "seedHi":3603515, + "seedLo":233373850, + "split":[ + 0.5, + 0.5 + ], + "trafficSeedHi":449867249, + "trafficSeedLo":455443629, + "trafficSplit":[ + 0.0, + 1.0 + ], + "fullOnVariant":0, + "applications":[ + { + "name":"website" + } + ], + "variants":[ + { + "name":"A", + "config":null + }, + { + "name":"B", + "config":"{\"banner.border\":1,\"banner.size\":\"large\"}" + } + ], + "audience": null, + "holdoutIds":[ + 11, + 12 + ] + } + ], + "holdouts":[ + { + "id":11, + "seedHi":13, + "seedLo":111, + "split":[ + 0.1, + 0.9 + ] + }, + { + "id":12, + "seedHi":1, + "seedLo":222, + "split":[ + 0.05, + 0.95 + ] + } + ] +} From 3e5318a3dd55eb9af662c399ed8c3e174fc614d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 4 Jul 2026 17:36:32 +0000 Subject: [PATCH 02/49] fix: post-restore review findings Add missing exposure assertion for the not-held-out-but-has-holdouts path (heldOut=false/holdoutId=0 after holdouts are actually evaluated), plus empty-holdoutIds and empty-split branch coverage. Co-Authored-By: Claude Opus 4.8 --- .../com/absmartly/sdk/ContextHoldoutTest.java | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 3b6e615..5b4fa64 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -382,4 +382,58 @@ void exposureCarriesHeldOutAndHoldoutId() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + + @Test + void exposureCarriesNotHeldOutFieldsWhenUnitNotInHoldout() { + final Experiment experiment = newExperiment(1, "exp_holdout_out_exposure"); + experiment.holdoutIds = new int[]{11}; + + // holdout evaluated but unit is NOT in it -> normal assignment, heldOut=false, holdoutId=0 + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_out_exposure")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_holdout_out_exposure", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false, false, 0), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void assignsNormallyWhenHoldoutIdsIsEmpty() { + final Experiment experiment = newExperiment(1, "exp_empty_holdout_ids"); + experiment.holdoutIds = new int[0]; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_empty_holdout_ids")); + } + + @Test + void skipsHoldoutWithEmptySplit() { + final Experiment experiment = newExperiment(1, "exp_empty_split_holdout"); + experiment.holdoutIds = new int[]{11}; + + // a holdout with an empty split must be dropped, not hold the unit out + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{new ExperimentHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO, + new double[0])}, + experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_empty_split_holdout")); + } } From c85fd0f8f6da662cd7682cfaa539db7a834d9322 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 4 Jul 2026 17:36:35 +0000 Subject: [PATCH 03/49] refactor: simplifier pass for holdouts Reuse the already-bound unitType local in the traffic-split branch instead of re-reading experiment.data.unitType, matching the new holdout branch. Behaviour-preserving. Co-Authored-By: Claude Opus 4.8 --- core-api/src/main/java/com/absmartly/sdk/Context.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index c22a90c..dfb5657 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -824,7 +824,7 @@ private Assignment getAssignment(final String experimentName) { if (experiment.data.audienceStrict && assignment.audienceMismatch) { assignment.variant = 0; } else if (experiment.data.fullOnVariant == 0) { - final String uid = units_.get(experiment.data.unitType); + final String uid = units_.get(unitType); if (uid != null) { final byte[] unitHash = Context.this.getUnitHash(unitType, uid); From 0efefdc5a673aef64f9011bc77eb9c93e4e3528b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 5 Jul 2026 09:10:32 +0000 Subject: [PATCH 04/49] style: spotless formatting --- .../com/absmartly/sdk/ContextHoldoutTest.java | 4 +-- .../java/com/absmartly/sdk/ContextTest.java | 36 ++++++++++++------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 5b4fa64..f291bb1 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1,7 +1,6 @@ package com.absmartly.sdk; import static org.junit.jupiter.api.Assertions.assertEquals; - import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -405,7 +404,8 @@ void exposureCarriesNotHeldOutFieldsWhenUnitNotInHoldout() { new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_holdout_out_exposure", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + new Exposure(1, "exp_holdout_out_exposure", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false, false, 0), }; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java index d63f473..3451fcc 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java @@ -577,7 +577,8 @@ void setUnitsBeforeReady() { expected.publishedAt = clock.millis(); expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -993,7 +994,8 @@ void getVariableValueQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1021,7 +1023,8 @@ void getVariableValueQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, + false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1050,7 +1053,8 @@ void getVariableValueCallsEventLogger() { context.getVariableValue("banner.size", null); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1124,14 +1128,16 @@ void getTreatment() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), new Exposure(2, "exp_test_abc", "session_id", 2, clock.millis(), true, true, false, false, false, false, false, 0), new Exposure(3, "exp_test_not_eligible", "user_id", 0, clock.millis(), true, false, false, false, false, false, false, 0), new Exposure(4, "exp_test_fullon", "session_id", 2, clock.millis(), true, true, false, true, false, false, false, 0), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, 0), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, + 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1266,7 +1272,8 @@ void getTreatmentQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1294,7 +1301,8 @@ void getTreatmentQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, + false, 0), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1344,8 +1352,10 @@ void getTreatmentCallsEventLogger() { context.getTreatment("not_found"); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, + 0), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1553,8 +1563,10 @@ void publishResetsInternalQueuesAndKeepsAttributesOverridesAndCustomAssignments( expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, false, 0), - new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, + false, 0), + new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false, + false, 0), new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false, false, 0), }; From 5af036aab343abe4aafadb9e5ba3386346e24cbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Thu, 9 Jul 2026 20:22:53 +0000 Subject: [PATCH 05/49] fix: address fable-5 review findings - Assignment cache: treat held-out assignments as up-to-date even when a custom assignment is set. Holdout wins over custom assignment, so the cached variant (0) can never equal the custom variant; the old condition forced a cache miss and a fresh Assignment (exposed=false) on every call, queueing a duplicate exposure per getTreatment. - Tests: cached held-out assignment with custom assignment set; holdout precedence over fullOnVariant. Co-Authored-By: Claude Fable 5 --- .../main/java/com/absmartly/sdk/Context.java | 2 +- .../com/absmartly/sdk/ContextHoldoutTest.java | 29 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index dfb5657..e5e32de 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -750,7 +750,7 @@ private Assignment getAssignment(final String experimentName) { // previously not-running experiment return assignment; } - } else if ((custom == null) || custom == assignment.variant) { + } else if ((custom == null) || assignment.heldOut || custom == assignment.variant) { if (experimentMatches(experiment, assignment)) { // assignment up-to-date return assignment; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index f291bb1..698f1c9 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -302,6 +302,35 @@ void holdoutTakesPrecedenceOverCustomAssignment() { assertEquals(0, context.peekTreatment("exp_holdout_custom")); } + @Test + void reusesCachedHeldOutAssignmentWithCustomAssignment() { + final Experiment experiment = newExperiment(1, "exp_holdout_custom_cache"); + experiment.holdoutIds = new int[]{11}; + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_holdout_custom_cache", 3); + final Context context = createReadyContext(config, contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + // the custom assignment can never apply while held out; repeated calls must hit the + // cache and not queue duplicate exposures + assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); + assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); + assertEquals(1, context.getPendingCount()); + } + + @Test + void holdoutTakesPrecedenceOverFullOn() { + final Experiment experiment = newExperiment(1, "exp_holdout_fullon"); + experiment.fullOnVariant = 2; + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_holdout_fullon")); + } + @Test void refreshReassignsWhenHoldoutsChange() { final Experiment experiment = newExperiment(1, "exp_holdout_refresh"); From 5ec87d45fed810ca431da448b37451bcfcd22462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 10 Jul 2026 14:28:05 +0000 Subject: [PATCH 06/49] fix: reuse cached forced-variant assignments with custom assignments When a custom assignment is set but the cached variant was forced by a higher-precedence rule (holdout, full-on, traffic ineligibility, or a strict audience mismatch), the custom value can never equal that variant. The old cache-validity check compared them directly, so it forced a cache miss and a fresh Assignment (exposed=false) on every getTreatment call, queueing a duplicate exposure each time. Introduce variantForcedRegardlessOfCustom() covering all forced-variant cases and treat them as cache-valid. 5af036a fixed the holdout case only; this generalizes the same fix to the audience-strict-mismatch and traffic-ineligible cases, which are pre-existing on main. Tests: cached forced-variant assignments with a custom assignment set for both the strict audience mismatch and traffic ineligible cases (stable pending count across repeated calls). Co-Authored-By: Claude Opus 4.8 --- .../main/java/com/absmartly/sdk/Context.java | 17 +++++++++- .../com/absmartly/sdk/ContextHoldoutTest.java | 34 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index e5e32de..8a9c0ae 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -704,6 +704,20 @@ private boolean experimentMatches(final ContextExperiment experiment, final Assi Arrays.equals(experiment.holdouts, assignment.holdouts); } + // A custom assignment can only take effect on the normal, traffic-eligible assignment path. + // When the cached variant was forced by a higher-precedence rule — a holdout, a full-on + // variant, traffic ineligibility, or a strict audience mismatch — the custom value can never + // equal that variant, so comparing the two would spuriously invalidate the cache and re-expose + // on every getTreatment call. Treat those forced assignments as cache-valid regardless of the + // custom assignment. (A held-out or forced assignment always has assigned=true except for the + // strict-mismatch and no-unit cases, where assigned stays false.) + private static boolean variantForcedRegardlessOfCustom(final Assignment assignment) { + return assignment.heldOut + || assignment.fullOn + || !assignment.eligible + || !assignment.assigned; + } + private static class Assignment { int id; int iteration; @@ -750,7 +764,8 @@ private Assignment getAssignment(final String experimentName) { // previously not-running experiment return assignment; } - } else if ((custom == null) || assignment.heldOut || custom == assignment.variant) { + } else if ((custom == null) || variantForcedRegardlessOfCustom(assignment) + || custom == assignment.variant) { if (experimentMatches(experiment, assignment)) { // assignment up-to-date return assignment; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 698f1c9..837118e 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -319,6 +319,40 @@ void reusesCachedHeldOutAssignmentWithCustomAssignment() { assertEquals(1, context.getPendingCount()); } + @Test + void reusesCachedAudienceMismatchAssignmentWithCustomAssignment() { + final Experiment experiment = newExperiment(1, "exp_audience_custom_cache"); + experiment.audienceStrict = true; + experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_audience_custom_cache", 3); + final Context context = createReadyContext(config, contextDataOf(experiment)); + context.setAttribute("age", 5); // mismatches the strict audience -> variant forced to 0 + + // strict audience mismatch forces variant 0; the custom assignment can never apply, so + // repeated calls must hit the cache and not queue duplicate exposures + assertEquals(0, context.getTreatment("exp_audience_custom_cache")); + assertEquals(0, context.getTreatment("exp_audience_custom_cache")); + assertEquals(1, context.getPendingCount()); + } + + @Test + void reusesCachedTrafficIneligibleAssignmentWithCustomAssignment() { + final Experiment experiment = newExperiment(1, "exp_traffic_custom_cache"); + experiment.trafficSplit = new double[]{1.0, 0.0}; // unit is NOT in the experiment traffic + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( + "exp_traffic_custom_cache", 3); + final Context context = createReadyContext(config, contextDataOf(experiment)); + + // traffic ineligibility forces variant 0; the custom assignment can never apply, so repeated + // calls must hit the cache and not queue duplicate exposures + assertEquals(0, context.getTreatment("exp_traffic_custom_cache")); + assertEquals(0, context.getTreatment("exp_traffic_custom_cache")); + assertEquals(1, context.getPendingCount()); + } + @Test void holdoutTakesPrecedenceOverFullOn() { final Experiment experiment = newExperiment(1, "exp_holdout_fullon"); From d15f26e22a1941c89894c35506c2de3f7deb8f62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 10 Jul 2026 14:28:14 +0000 Subject: [PATCH 07/49] docs: document shared-array immutability invariant Assignment.holdoutIds/holdouts alias the arrays owned by ContextExperiment instead of copying them. This is safe only because experiment data is treated as immutable after setData and the arrays are read-only here. Note the invariant at the assignment site. Co-Authored-By: Claude Opus 4.8 --- core-api/src/main/java/com/absmartly/sdk/Context.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 8a9c0ae..ba48f4b 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -801,6 +801,9 @@ private Assignment getAssignment(final String experimentName) { if (experiment != null) { final String unitType = experiment.data.unitType; + // Share the experiment's holdout arrays by reference rather than copying: they are + // only read here (and via Arrays.equals in experimentMatches) and experiment data is + // treated as immutable once installed by setData, so the aliasing is safe. assignment.holdoutIds = experiment.data.holdoutIds; assignment.holdouts = experiment.holdouts; From b2e54ed9071297e3bc0ca7bbc38f94623c12e9c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 12 Jul 2026 21:39:38 +0000 Subject: [PATCH 08/49] fix: address gpt-5.6-sol review findings --- .../main/java/com/absmartly/sdk/Context.java | 7 +++++ .../sdk/internal/VariantAssigner.java | 8 ++++- .../com/absmartly/sdk/ContextHoldoutTest.java | 30 +++++++++++++++++++ .../sdk/internal/VariantAssignerTest.java | 6 ++++ 4 files changed, 50 insertions(+), 1 deletion(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index ba48f4b..52e20e4 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1021,6 +1021,13 @@ private static ExperimentHoldout[] resolveHoldouts(final int[] holdoutIds, return null; } + Collections.sort(resolved, new Comparator() { + @Override + public int compare(ExperimentHoldout a, ExperimentHoldout b) { + return Integer.valueOf(a.id).compareTo(b.id); + } + }); + return resolved.toArray(new ExperimentHoldout[0]); } diff --git a/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java b/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java index 52435ac..0aaaa8f 100644 --- a/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java +++ b/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java @@ -9,7 +9,9 @@ public byte[] initialValue() { return new byte[12]; } }; - private static final double normalizer = 1.0 / 0xffffffffL; + // Convert an unsigned 32-bit hash to [0, 1). Dividing by 0xffffffff would + // produce exactly 1.0 for one hash value, incorrectly missing a 100% split. + private static final double normalizer = 1.0 / 0x100000000L; public VariantAssigner(byte[] unitHash) { unitHash_ = Murmur3_32.digest(unitHash, 0); @@ -40,6 +42,10 @@ private double probability(int seedHi, int seedLo) { Buffers.putUInt32(buffer, 8, unitHash_); final int hash = Murmur3_32.digest(buffer, 0); + return normalizeHash(hash); + } + + static double normalizeHash(int hash) { return (hash & 0xffffffffL) * normalizer; } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 837118e..5f25a8c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -154,6 +154,36 @@ void checksAllHoldoutsUntilAMatchIsFound() { assertEquals(0, context.peekTreatment("exp_multi_holdout")); } + @Test + void evaluatesMultipleHoldoutsInCanonicalIdOrder() { + final Experiment experiment = newExperiment(1, "exp_ordered_holdouts"); + experiment.holdoutIds = new int[]{42, 11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{ + newHoldout(42, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + }, experiment)); + + assertEquals(0, context.getTreatment("exp_ordered_holdouts")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_ordered_holdouts", UNIT_TYPE, 0, clock.millis(), true, true, false, false, + false, false, true, 11), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + @Test void assignsNormallyWhenNotInAnyHoldout() { final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); diff --git a/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java b/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java index 58ffba9..16ddd94 100644 --- a/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java @@ -14,6 +14,12 @@ import com.absmartly.sdk.internal.hashing.Hashing; class VariantAssignerTest extends TestUtils { + @Test + void normalizeHashProducesHalfOpenProbabilityRange() { + assertEquals(0.0, VariantAssigner.normalizeHash(0)); + assertEquals(1.0 - (1.0 / 0x100000000L), VariantAssigner.normalizeHash(-1)); + } + @Test void chooseVariant() { assertEquals(1, VariantAssigner.chooseVariant(new double[]{0.0, 1.0}, 0.0)); From c2a8338bbca3872933830993b6f5699c6c29ee86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 13 Jul 2026 09:43:35 +0000 Subject: [PATCH 09/49] fix: revert VariantAssigner normalizer change to preserve collector hash parity The gpt-5.6-sol pass changed the SDK normalizer to 1.0/0x100000000 to make the probability range half-open [0,1). But the collector (and every other ABsmartly SDK) uses 1.0/0xffffffff, where the max 32-bit hash maps to exactly 1.0. Assignment requires bit-exact hash parity between the SDK and the collector -- diverging the normalizer would make the SDK and server disagree on borderline units (including holdout membership). Reverted VariantAssigner to match main/collector; removed the range test that asserted the divergent behaviour. --- .../java/com/absmartly/sdk/internal/VariantAssigner.java | 8 +------- .../com/absmartly/sdk/internal/VariantAssignerTest.java | 6 ------ 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java b/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java index 0aaaa8f..52435ac 100644 --- a/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java +++ b/core-api/src/main/java/com/absmartly/sdk/internal/VariantAssigner.java @@ -9,9 +9,7 @@ public byte[] initialValue() { return new byte[12]; } }; - // Convert an unsigned 32-bit hash to [0, 1). Dividing by 0xffffffff would - // produce exactly 1.0 for one hash value, incorrectly missing a 100% split. - private static final double normalizer = 1.0 / 0x100000000L; + private static final double normalizer = 1.0 / 0xffffffffL; public VariantAssigner(byte[] unitHash) { unitHash_ = Murmur3_32.digest(unitHash, 0); @@ -42,10 +40,6 @@ private double probability(int seedHi, int seedLo) { Buffers.putUInt32(buffer, 8, unitHash_); final int hash = Murmur3_32.digest(buffer, 0); - return normalizeHash(hash); - } - - static double normalizeHash(int hash) { return (hash & 0xffffffffL) * normalizer; } diff --git a/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java b/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java index 16ddd94..58ffba9 100644 --- a/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/internal/VariantAssignerTest.java @@ -14,12 +14,6 @@ import com.absmartly.sdk.internal.hashing.Hashing; class VariantAssignerTest extends TestUtils { - @Test - void normalizeHashProducesHalfOpenProbabilityRange() { - assertEquals(0.0, VariantAssigner.normalizeHash(0)); - assertEquals(1.0 - (1.0 / 0x100000000L), VariantAssigner.normalizeHash(-1)); - } - @Test void chooseVariant() { assertEquals(1, VariantAssigner.chooseVariant(new double[]{0.0, 1.0}, 0.0)); From eb7dd64ccbeaba7043fb3cd5a213fb2609706c7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Thu, 16 Jul 2026 16:18:18 +0000 Subject: [PATCH 10/49] feat: support full_on holdouts Add optional fullOn field to ExperimentHoldout (tolerant, defaults to null/absent = full holdout, unchanged behavior). A full_on holdout only applies to full-on experiments (fullOnVariant != 0); for other experiments the SDK skips it entirely during heldOut evaluation, matching the collector's server-side behavior bit-exactly. --- .../main/java/com/absmartly/sdk/Context.java | 6 + .../absmartly/sdk/json/ExperimentHoldout.java | 11 ++ .../com/absmartly/sdk/ContextHoldoutTest.java | 109 ++++++++++++++++++ 3 files changed, 126 insertions(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 52e20e4..6408386 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -814,6 +814,12 @@ private Assignment getAssignment(final String experimentName) { final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); for (final ExperimentHoldout holdout : experiment.holdouts) { + if (Boolean.TRUE.equals(holdout.fullOn) && experiment.data.fullOnVariant == 0) { + // a full_on holdout only applies to full-on experiments; the + // collector skips it server-side for non-full-on experiments too. + continue; + } + if (assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo) == 0) { assignment.heldOut = true; assignment.holdoutId = holdout.id; diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java b/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java index cbaf4e6..c17bd27 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java @@ -14,15 +14,22 @@ public class ExperimentHoldout { public int seedHi; public int seedLo; public double[] split; + public Boolean fullOn; public ExperimentHoldout() {} @SuppressFBWarnings(value = "EI_EXPOSE_REP2") public ExperimentHoldout(int id, int seedHi, int seedLo, double[] split) { + this(id, seedHi, seedLo, split, null); + } + + @SuppressFBWarnings(value = "EI_EXPOSE_REP2") + public ExperimentHoldout(int id, int seedHi, int seedLo, double[] split, Boolean fullOn) { this.id = id; this.seedHi = seedHi; this.seedLo = seedLo; this.split = split; + this.fullOn = fullOn; } @Override @@ -40,6 +47,8 @@ public boolean equals(Object o) { return false; if (seedLo != that.seedLo) return false; + if (fullOn != null ? !fullOn.equals(that.fullOn) : that.fullOn != null) + return false; return Arrays.equals(split, that.split); } @@ -49,6 +58,7 @@ public int hashCode() { result = 31 * result + seedHi; result = 31 * result + seedLo; result = 31 * result + Arrays.hashCode(split); + result = 31 * result + (fullOn != null ? fullOn.hashCode() : 0); return result; } @@ -59,6 +69,7 @@ public String toString() { ", seedHi=" + seedHi + ", seedLo=" + seedLo + ", split=" + Arrays.toString(split) + + ", fullOn=" + fullOn + '}'; } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 5f25a8c..812f779 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -98,6 +98,10 @@ static ExperimentHoldout newHoldout(int id, int seedHi, int seedLo) { return new ExperimentHoldout(id, seedHi, seedLo, new double[]{0.1, 0.9}); } + static ExperimentHoldout newFullOnHoldout(int id, int seedHi, int seedLo) { + return new ExperimentHoldout(id, seedHi, seedLo, new double[]{0.1, 0.9}, true); + } + static ContextData contextDataOf(Experiment... experiments) { return contextDataOf(new ExperimentHoldout[0], experiments); } @@ -529,4 +533,109 @@ void skipsHoldoutWithEmptySplit() { assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_empty_split_holdout")); } + + @Test + void fullOnHoldoutAppliesToFullOnExperiment() { + final Experiment experiment = newExperiment(1, "exp_fullon_holdout_fullon_exp"); + experiment.fullOnVariant = 2; + experiment.holdoutIds = new int[]{11}; + + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_fullon_holdout_fullon_exp")); + } + + @Test + void fullOnHoldoutIsSkippedForRegularExperiment() { + final Experiment experiment = newExperiment(1, "exp_fullon_holdout_regular_exp"); + experiment.holdoutIds = new int[]{11}; + + // the unit would be held out (matches the holdout's split), but fullOn holdouts only + // apply to full-on experiments (fullOnVariant != 0), so it must be skipped entirely and + // normal assignment must proceed. + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_fullon_holdout_regular_exp")); + } + + @Test + void absentFullOnBehavesAsFullHoldoutOnFullOnExperiment() { + final Experiment experiment = newExperiment(1, "exp_absent_fullon_holdout_fullon_exp"); + experiment.fullOnVariant = 2; + experiment.holdoutIds = new int[]{11}; + + // fullOn absent (null) -> treated as a regular (full) holdout, applies regardless of + // whether the experiment is full-on. + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + + assertEquals(0, context.peekTreatment("exp_absent_fullon_holdout_fullon_exp")); + } + + @Test + void mixedFullAndFullOnHoldoutsOnRegularExperiment() { + final Experiment experiment = newExperiment(1, "exp_mixed_full_fullon_regular"); + experiment.holdoutIds = new int[]{11, 12}; + + // 11 is a full_on holdout that would match but must be skipped (regular experiment); + // 12 is a regular (full) holdout that matches -> unit is held out via 12. + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{ + newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + }, experiment)); + + assertEquals(0, context.getTreatment("exp_mixed_full_fullon_regular")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_mixed_full_fullon_regular", UNIT_TYPE, 0, clock.millis(), true, true, false, + false, false, false, true, 12), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void mixedFullAndFullOnHoldoutsOnFullOnExperiment() { + final Experiment experiment = newExperiment(1, "exp_mixed_full_fullon_fullon_exp"); + experiment.fullOnVariant = 2; + experiment.holdoutIds = new int[]{11, 12}; + + // on a full-on experiment both holdouts apply; iteration order is by payload id, so the + // full_on holdout (11) matches first. + final Context context = createReadyContext(contextDataOf( + new ExperimentHoldout[]{ + newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), + }, experiment)); + + assertEquals(0, context.getTreatment("exp_mixed_full_fullon_fullon_exp")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) + }; + expected.exposures = new Exposure[]{ + new Exposure(1, "exp_mixed_full_fullon_fullon_exp", UNIT_TYPE, 0, clock.millis(), true, true, false, + false, false, false, true, 11), + }; + + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } } From 4fc32ad5f9064cbd0d09c25b396de16d9dba9f40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 8 Aug 2026 23:19:39 +0000 Subject: [PATCH 11/49] refactor: revert holdout wire model to first-class experiment shape Holdouts arrive as ordinary Experiment entries in ContextData.holdouts, carrying holdoutType (full/full_on) and excludedExperimentIds instead of a separate ExperimentHoldout model. Experiment.holdoutIds is removed: applicability is derived client-side from unit type and the full/full_on rule, not served per-experiment. Exposure drops heldOut/holdoutId, which no longer exist server-side; a held-out unit emits no exposure for the experiments it covers instead of a tagged one. --- .../com/absmartly/sdk/json/ContextData.java | 4 +- .../com/absmartly/sdk/json/Experiment.java | 15 +++- .../absmartly/sdk/json/ExperimentHoldout.java | 75 ------------------- .../java/com/absmartly/sdk/json/Exposure.java | 14 +--- .../java/com/absmartly/sdk/ContextTest.java | 56 ++++++-------- .../DefaultContextEventSerializerTest.java | 5 +- .../sdk/json/ExperimentHoldoutTest.java | 31 -------- 7 files changed, 40 insertions(+), 160 deletions(-) delete mode 100644 core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java delete mode 100644 core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java index 9833f2e..0944e53 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/ContextData.java @@ -11,7 +11,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) public class ContextData { public Experiment[] experiments = new Experiment[0]; - public ExperimentHoldout[] holdouts = new ExperimentHoldout[0]; + public Experiment[] holdouts = new Experiment[0]; public ContextData() {} @@ -21,7 +21,7 @@ public ContextData(Experiment[] experiments) { } @SuppressFBWarnings(value = "EI_EXPOSE_REP2") - public ContextData(Experiment[] experiments, ExperimentHoldout[] holdouts) { + public ContextData(Experiment[] experiments, Experiment[] holdouts) { this.experiments = experiments; this.holdouts = holdouts; } diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java index e3a8459..a8d2f24 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java @@ -24,7 +24,10 @@ public class Experiment { public boolean audienceStrict; public String audience; public CustomFieldValue[] customFieldValues; - public int[] holdoutIds; + + // Set only for entries served in the top-level `holdouts` array; null for ordinary experiments. + public String holdoutType; + public int[] excludedExperimentIds; public Experiment() {} @@ -69,7 +72,9 @@ public boolean equals(Object o) { return false; if (!Arrays.equals(customFieldValues, that.customFieldValues)) return false; - return Arrays.equals(holdoutIds, that.holdoutIds); + if (holdoutType != null ? !holdoutType.equals(that.holdoutType) : that.holdoutType != null) + return false; + return Arrays.equals(excludedExperimentIds, that.excludedExperimentIds); } @Override @@ -90,7 +95,8 @@ public int hashCode() { result = 31 * result + (audienceStrict ? 1 : 0); result = 31 * result + (audience != null ? audience.hashCode() : 0); result = 31 * result + Arrays.hashCode(customFieldValues); - result = 31 * result + Arrays.hashCode(holdoutIds); + result = 31 * result + (holdoutType != null ? holdoutType.hashCode() : 0); + result = 31 * result + Arrays.hashCode(excludedExperimentIds); return result; } @@ -113,7 +119,8 @@ public String toString() { ", audienceStrict=" + audienceStrict + ", audience='" + audience + '\'' + ", customFieldValues=" + Arrays.toString(customFieldValues) + - ", holdoutIds=" + Arrays.toString(holdoutIds) + + ", holdoutType='" + holdoutType + '\'' + + ", excludedExperimentIds=" + Arrays.toString(excludedExperimentIds) + '}'; } } diff --git a/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java b/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java deleted file mode 100644 index c17bd27..0000000 --- a/core-api/src/main/java/com/absmartly/sdk/json/ExperimentHoldout.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.absmartly.sdk.json; - -import java.util.Arrays; - -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; - -import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; - -@JsonInclude(JsonInclude.Include.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -public class ExperimentHoldout { - public int id; - public int seedHi; - public int seedLo; - public double[] split; - public Boolean fullOn; - - public ExperimentHoldout() {} - - @SuppressFBWarnings(value = "EI_EXPOSE_REP2") - public ExperimentHoldout(int id, int seedHi, int seedLo, double[] split) { - this(id, seedHi, seedLo, split, null); - } - - @SuppressFBWarnings(value = "EI_EXPOSE_REP2") - public ExperimentHoldout(int id, int seedHi, int seedLo, double[] split, Boolean fullOn) { - this.id = id; - this.seedHi = seedHi; - this.seedLo = seedLo; - this.split = split; - this.fullOn = fullOn; - } - - @Override - public boolean equals(Object o) { - if (this == o) - return true; - if (o == null || getClass() != o.getClass()) - return false; - - ExperimentHoldout that = (ExperimentHoldout) o; - - if (id != that.id) - return false; - if (seedHi != that.seedHi) - return false; - if (seedLo != that.seedLo) - return false; - if (fullOn != null ? !fullOn.equals(that.fullOn) : that.fullOn != null) - return false; - return Arrays.equals(split, that.split); - } - - @Override - public int hashCode() { - int result = id; - result = 31 * result + seedHi; - result = 31 * result + seedLo; - result = 31 * result + Arrays.hashCode(split); - result = 31 * result + (fullOn != null ? fullOn.hashCode() : 0); - return result; - } - - @Override - public String toString() { - return "ExperimentHoldout{" + - "id=" + id + - ", seedHi=" + seedHi + - ", seedLo=" + seedLo + - ", split=" + Arrays.toString(split) + - ", fullOn=" + fullOn + - '}'; - } -} diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java b/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java index 6908582..5c0e158 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Exposure.java @@ -19,14 +19,11 @@ public class Exposure { public boolean fullOn; public boolean custom; public boolean audienceMismatch; - public boolean heldOut; - public int holdoutId; public Exposure() {} public Exposure(int id, String name, String unit, int variant, long exposedAt, boolean assigned, boolean eligible, - boolean overridden, boolean fullOn, boolean custom, boolean audienceMismatch, boolean heldOut, - int holdoutId) { + boolean overridden, boolean fullOn, boolean custom, boolean audienceMismatch) { this.id = id; this.name = name; this.unit = unit; @@ -38,8 +35,6 @@ public Exposure(int id, String name, String unit, int variant, long exposedAt, b this.fullOn = fullOn; this.custom = custom; this.audienceMismatch = audienceMismatch; - this.heldOut = heldOut; - this.holdoutId = holdoutId; } @Override @@ -53,14 +48,13 @@ public boolean equals(Object o) { && assigned == exposure.assigned && eligible == exposure.eligible && overridden == exposure.overridden && fullOn == exposure.fullOn && custom == exposure.custom && Objects.equals(audienceMismatch, exposure.audienceMismatch) && Objects.equals(name, exposure.name) - && Objects.equals(unit, exposure.unit) && heldOut == exposure.heldOut - && holdoutId == exposure.holdoutId; + && Objects.equals(unit, exposure.unit); } @Override public int hashCode() { return Objects.hash(id, name, unit, variant, exposedAt, assigned, eligible, overridden, fullOn, custom, - audienceMismatch, heldOut, holdoutId); + audienceMismatch); } @Override @@ -77,8 +71,6 @@ public String toString() { ", fullOn=" + fullOn + ", custom=" + custom + ", audienceMismatch=" + audienceMismatch + - ", heldOut=" + heldOut + - ", holdoutId=" + holdoutId + '}'; } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java index 3451fcc..0abe1c0 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java @@ -577,8 +577,7 @@ void setUnitsBeforeReady() { expected.publishedAt = clock.millis(); expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -994,8 +993,7 @@ void getVariableValueQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1023,8 +1021,7 @@ void getVariableValueQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1053,8 +1050,7 @@ void getVariableValueCallsEventLogger() { context.getVariableValue("banner.size", null); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1128,16 +1124,14 @@ void getTreatment() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), new Exposure(2, "exp_test_abc", "session_id", 2, clock.millis(), true, true, false, false, false, - false, false, 0), + false), new Exposure(3, "exp_test_not_eligible", "user_id", 0, clock.millis(), true, false, false, false, - false, false, false, 0), + false, false), new Exposure(4, "exp_test_fullon", "session_id", 2, clock.millis(), true, true, false, true, false, - false, false, 0), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, - 0), + false), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1201,14 +1195,14 @@ void getTreatmentReturnsOverrideVariant() { expected.exposures = new Exposure[]{ new Exposure(1, "exp_test_ab", "session_id", 12, clock.millis(), false, true, true, false, false, - false, false, 0), + false), new Exposure(2, "exp_test_abc", "session_id", 13, clock.millis(), false, true, true, false, false, - false, false, 0), + false), new Exposure(3, "exp_test_not_eligible", "user_id", 11, clock.millis(), false, true, true, false, false, - false, false, 0), + false), new Exposure(4, "exp_test_fullon", "session_id", 13, clock.millis(), false, true, true, false, false, - false, false, 0), - new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false, false, 0), + false), + new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1272,8 +1266,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchFalseOnAudienceMatch() { }; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1301,8 +1294,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchTrueOnAudienceMismatch() { expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, true), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1331,7 +1323,7 @@ void getTreatmentQueuesExposureWithAudienceMismatchTrueAndControlVariantOnAudien expected.exposures = new Exposure[]{ new Exposure(1, "exp_test_ab", "session_id", 0, clock.millis(), false, true, false, false, false, - true, false, 0), + true), }; when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); @@ -1352,10 +1344,8 @@ void getTreatmentCallsEventLogger() { context.getTreatment("not_found"); final Exposure[] exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), - new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false, false, - 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(0, "not_found", null, 0, clock.millis(), false, true, false, false, false, false), }; verify(eventLogger, Mockito.timeout(5000).times(exposures.length)).handleEvent(any(), any(), any()); @@ -1563,11 +1553,9 @@ void publishResetsInternalQueuesAndKeepsAttributesOverridesAndCustomAssignments( expected.units = publishUnits; expected.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false, - false, 0), - new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false, - false, 0), - new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false, false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, clock.millis(), true, true, false, false, false, false), + new Exposure(2, "exp_test_abc", "session_id", 3, clock.millis(), true, true, false, false, true, false), + new Exposure(0, "not_found", null, 3, clock.millis(), false, true, true, false, false, false), }; expected.goals = new GoalAchievement[]{ diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java index 893b307..5c0dc59 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextEventSerializerTest.java @@ -34,8 +34,7 @@ void serialize() { "nested_arr", mapOf("nested", listOf(1, 2, "test"))); event.exposures = new Exposure[]{ - new Exposure(1, "exp_test_ab", "session_id", 1, 123470000L, true, true, false, false, false, true, - false, 0), + new Exposure(1, "exp_test_ab", "session_id", 1, 123470000L, true, true, false, false, false, true), }; event.goals = new GoalAchievement[]{ @@ -55,7 +54,7 @@ void serialize() { final byte[] bytes = ser.serialize(event); assertEquals( - "{\"hashed\":true,\"units\":[{\"type\":\"session_id\",\"uid\":\"pAE3a1i5Drs5mKRNq56adA\"},{\"type\":\"user_id\",\"uid\":\"JfnnlDI7RTiF9RgfG2JNCw\"}],\"publishedAt\":123456789,\"exposures\":[{\"id\":1,\"name\":\"exp_test_ab\",\"unit\":\"session_id\",\"variant\":1,\"exposedAt\":123470000,\"assigned\":true,\"eligible\":true,\"overridden\":false,\"fullOn\":false,\"custom\":false,\"audienceMismatch\":true,\"heldOut\":false,\"holdoutId\":0}],\"goals\":[{\"name\":\"goal1\",\"achievedAt\":123456000,\"properties\":{\"amount\":6,\"nested\":{\"value\":5},\"nested_arr\":{\"nested\":[1,2,\"test\"]},\"tries\":1,\"value\":5.0}},{\"name\":\"goal2\",\"achievedAt\":123456789}],\"attributes\":[{\"name\":\"attr1\",\"value\":\"value1\",\"setAt\":123456000},{\"name\":\"attr2\",\"value\":\"value2\",\"setAt\":123456789},{\"name\":\"attr2\",\"setAt\":123450000},{\"name\":\"attr3\",\"value\":{\"nested\":{\"value\":5}},\"setAt\":123470000},{\"name\":\"attr4\",\"value\":{\"nested\":[1,2,\"test\"]},\"setAt\":123480000}]}", + "{\"hashed\":true,\"units\":[{\"type\":\"session_id\",\"uid\":\"pAE3a1i5Drs5mKRNq56adA\"},{\"type\":\"user_id\",\"uid\":\"JfnnlDI7RTiF9RgfG2JNCw\"}],\"publishedAt\":123456789,\"exposures\":[{\"id\":1,\"name\":\"exp_test_ab\",\"unit\":\"session_id\",\"variant\":1,\"exposedAt\":123470000,\"assigned\":true,\"eligible\":true,\"overridden\":false,\"fullOn\":false,\"custom\":false,\"audienceMismatch\":true}],\"goals\":[{\"name\":\"goal1\",\"achievedAt\":123456000,\"properties\":{\"amount\":6,\"nested\":{\"value\":5},\"nested_arr\":{\"nested\":[1,2,\"test\"]},\"tries\":1,\"value\":5.0}},{\"name\":\"goal2\",\"achievedAt\":123456789}],\"attributes\":[{\"name\":\"attr1\",\"value\":\"value1\",\"setAt\":123456000},{\"name\":\"attr2\",\"value\":\"value2\",\"setAt\":123456789},{\"name\":\"attr2\",\"setAt\":123450000},{\"name\":\"attr3\",\"value\":{\"nested\":{\"value\":5}},\"setAt\":123470000},{\"name\":\"attr4\",\"value\":{\"nested\":[1,2,\"test\"]},\"setAt\":123480000}]}", new String(bytes, StandardCharsets.UTF_8)); } diff --git a/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java deleted file mode 100644 index 8176ddd..0000000 --- a/core-api/src/test/java/com/absmartly/sdk/json/ExperimentHoldoutTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.absmartly.sdk.json; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import org.junit.jupiter.api.Test; - -class ExperimentHoldoutTest { - @Test - void equalsHashCodeAndToString() { - final ExperimentHoldout a = new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}); - final ExperimentHoldout b = new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}); - - assertEquals(a, a); - assertEquals(a, b); - assertEquals(a.hashCode(), b.hashCode()); - - assertNotEquals(a, null); - assertNotEquals(a, "not a holdout"); - assertNotEquals(a, new ExperimentHoldout(99, 13, 111, new double[]{0.1, 0.9})); - assertNotEquals(a, new ExperimentHoldout(11, 99, 111, new double[]{0.1, 0.9})); - assertNotEquals(a, new ExperimentHoldout(11, 13, 999, new double[]{0.1, 0.9})); - assertNotEquals(a, new ExperimentHoldout(11, 13, 111, new double[]{0.2, 0.8})); - - final String text = a.toString(); - assertTrue(text.contains("id=11")); - assertTrue(text.contains("seedHi=13")); - assertTrue(text.contains("seedLo=111")); - } -} From 386a80ca6306299dbf36ece4d0e1a14c1e624574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 00:06:12 +0000 Subject: [PATCH 12/49] feat: enforce holdout suppression and unified holdout exposure emission Suppress a covered experiment's own exposure and force control values whenever the unit is variant 0 in any applicable holdout, instead of tagging the exposure with the held-out holdout's id. Applicability is now derived per experiment at data-install time from unit type, full_on/fullOnVariant, and excludedExperimentIds, since holdouts are served as ordinary experiment entries rather than a separate id-keyed model. The holdout experiment itself is assigned and cached like any other experiment (by holdout id, since it lives outside the experiments index) and emits one ordinary exposure the first time any experiment applicable to its unit type is evaluated, regardless of whether that triggering experiment's own exposure is suppressed. --- .../main/java/com/absmartly/sdk/Context.java | 233 +++++++++++++----- 1 file changed, 170 insertions(+), 63 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 6408386..9f2ebcb 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -378,37 +378,58 @@ public int getTreatment(@Nonnull final String experimentName) { return assignment.variant; } + // A suppressed assignment (held out by an applicable holdout) never publishes its own + // exposure - the holdout experiment's own exposure is the sole membership record. Firing + // either exposure still triggers evaluation of every holdout applicable to this unit type, + // keeping both holdout arms symmetric regardless of which covered experiment triggered it. private void queueExposure(final Assignment assignment) { if (assignment.exposed.compareAndSet(false, true)) { - final Exposure exposure = new Exposure(); - exposure.id = assignment.id; - exposure.name = assignment.name; - exposure.unit = assignment.unitType; - exposure.variant = assignment.variant; - exposure.exposedAt = clock_.millis(); - exposure.assigned = assignment.assigned; - exposure.eligible = assignment.eligible; - exposure.overridden = assignment.overridden; - exposure.fullOn = assignment.fullOn; - exposure.custom = assignment.custom; - exposure.audienceMismatch = assignment.audienceMismatch; - exposure.heldOut = assignment.heldOut; - exposure.holdoutId = assignment.holdoutId; - - try { - eventLock_.lock(); - pendingCount_.incrementAndGet(); - exposures_.add(exposure); - } finally { - eventLock_.unlock(); + if (!assignment.suppressed) { + enqueueExposure(assignment); } - logEvent(ContextEventLogger.EventType.Exposure, exposure); + if (assignment.holdouts != null) { + for (final Experiment holdout : assignment.holdouts) { + queueHoldoutExposure(holdout, assignment.unitType); + } + } setTimeout(); } } + private void queueHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { + final Assignment holdoutAssignment = getHoldoutAssignment(holdoutExperiment, unitType); + if ((holdoutAssignment != null) && holdoutAssignment.exposed.compareAndSet(false, true)) { + enqueueExposure(holdoutAssignment); + } + } + + private void enqueueExposure(final Assignment assignment) { + final Exposure exposure = new Exposure(); + exposure.id = assignment.id; + exposure.name = assignment.name; + exposure.unit = assignment.unitType; + exposure.variant = assignment.variant; + exposure.exposedAt = clock_.millis(); + exposure.assigned = assignment.assigned; + exposure.eligible = assignment.eligible; + exposure.overridden = assignment.overridden; + exposure.fullOn = assignment.fullOn; + exposure.custom = assignment.custom; + exposure.audienceMismatch = assignment.audienceMismatch; + + try { + eventLock_.lock(); + pendingCount_.incrementAndGet(); + exposures_.add(exposure); + } finally { + eventLock_.unlock(); + } + + logEvent(ContextEventLogger.EventType.Exposure, exposure); + } + public int peekTreatment(@Nonnull final String experimentName) { checkReady(true); @@ -700,7 +721,6 @@ private boolean experimentMatches(final ContextExperiment experiment, final Assi experiment.data.iteration == assignment.iteration && experiment.data.fullOnVariant == assignment.fullOnVariant && Arrays.equals(experiment.data.trafficSplit, assignment.trafficSplit) && - Arrays.equals(experiment.data.holdoutIds, assignment.holdoutIds) && Arrays.equals(experiment.holdouts, assignment.holdouts); } @@ -709,10 +729,10 @@ private boolean experimentMatches(final ContextExperiment experiment, final Assi // variant, traffic ineligibility, or a strict audience mismatch — the custom value can never // equal that variant, so comparing the two would spuriously invalidate the cache and re-expose // on every getTreatment call. Treat those forced assignments as cache-valid regardless of the - // custom assignment. (A held-out or forced assignment always has assigned=true except for the + // custom assignment. (A suppressed or forced assignment always has assigned=true except for the // strict-mismatch and no-unit cases, where assigned stays false.) private static boolean variantForcedRegardlessOfCustom(final Assignment assignment) { - return assignment.heldOut + return assignment.suppressed || assignment.fullOn || !assignment.eligible || !assignment.assigned; @@ -733,15 +753,44 @@ private static class Assignment { boolean custom; boolean audienceMismatch; - boolean heldOut; - int holdoutId; - int[] holdoutIds; - ExperimentHoldout[] holdouts; + // Held out by a union of applicable holdouts: no exposure for this experiment, control + // values only. `holdouts` is the resolved applicable list, used both to invalidate this + // cached assignment when a holdout definition changes and to trigger each holdout's own + // exposure once this experiment is evaluated (see queueExposure). + boolean suppressed; + Experiment[] holdouts; Map variables = Collections.emptyMap(); final AtomicBoolean exposed = new AtomicBoolean(false); } + // Pins the holdout definition an Assignment was computed against, so getHoldoutAssignment can + // detect a refresh that changed the holdout's seed/split/iteration and recompute rather than + // reuse a stale membership verdict. Compares assignment-relevant fields only, mirroring + // experimentMatches for ordinary experiments. + private static class HoldoutAssignment { + final Assignment assignment; + final int iteration; + final int seedHi; + final int seedLo; + final double[] split; + final String unitType; + + HoldoutAssignment(Assignment assignment, Experiment holdout, String unitType) { + this.assignment = assignment; + this.iteration = holdout.iteration; + this.seedHi = holdout.seedHi; + this.seedLo = holdout.seedLo; + this.split = holdout.split; + this.unitType = unitType; + } + + boolean matches(Experiment current, String currentUnitType) { + return (iteration == current.iteration) && (seedHi == current.seedHi) && (seedLo == current.seedLo) + && Arrays.equals(split, current.split) && unitType.equals(currentUnitType); + } + } + private Assignment getAssignment(final String experimentName) { final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); try { @@ -801,37 +850,35 @@ private Assignment getAssignment(final String experimentName) { if (experiment != null) { final String unitType = experiment.data.unitType; - // Share the experiment's holdout arrays by reference rather than copying: they are - // only read here (and via Arrays.equals in experimentMatches) and experiment data is - // treated as immutable once installed by setData, so the aliasing is safe. - assignment.holdoutIds = experiment.data.holdoutIds; + // Share the experiment's applicable-holdouts array by reference rather than + // copying: it is only read here (and via Arrays.equals in experimentMatches) and + // experiment data is treated as immutable once installed by setData, so the + // aliasing is safe. assignment.holdouts = experiment.holdouts; + boolean suppressed = false; if (experiment.holdouts != null && experiment.holdouts.length > 0) { final String uid = units_.get(unitType); if (uid != null) { final byte[] unitHash = Context.this.getUnitHash(unitType, uid); final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); - for (final ExperimentHoldout holdout : experiment.holdouts) { - if (Boolean.TRUE.equals(holdout.fullOn) && experiment.data.fullOnVariant == 0) { - // a full_on holdout only applies to full-on experiments; the - // collector skips it server-side for non-full-on experiments too. - continue; - } - + // Union across every applicable holdout: variant 0 in any one of them + // suppresses this experiment's own exposure and forces control values. + for (final Experiment holdout : experiment.holdouts) { if (assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo) == 0) { - assignment.heldOut = true; - assignment.holdoutId = holdout.id; - assignment.variant = 0; - assignment.assigned = true; + suppressed = true; break; } } } } + assignment.suppressed = suppressed; - if (!assignment.heldOut) { + if (suppressed) { + assignment.variant = 0; + assignment.assigned = true; + } else { if (experiment.data.audience != null && experiment.data.audience.length() > 0) { final Map attrs = new HashMap(attributes_.size()); for (final Attribute attr : attributes_) { @@ -922,6 +969,46 @@ private ContextExperiment getExperiment(final String experimentName) { } } + // The holdout's own assignment is cached by holdout id rather than name, since holdout + // entries live outside the experiments index and are shared by reference across every + // covered experiment. A definition change (seed, split, iteration) invalidates the cache + // entry so a refreshed holdout is re-assigned and re-exposed, exactly like a normal + // experiment's cached assignment does via experimentMatches. + private Assignment getHoldoutAssignment(final Experiment holdout, final String unitType) { + final String uid = units_.get(unitType); + if (uid == null) { + return null; + } + + final ReentrantReadWriteLock.WriteLock writeLock = contextLock_.writeLock(); + try { + writeLock.lock(); + + final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); + if ((cached != null) && cached.matches(holdout, unitType)) { + return cached.assignment; + } + + final byte[] unitHash = Context.this.getUnitHash(unitType, uid); + final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); + + final Assignment assignment = new Assignment(); + assignment.id = holdout.id; + assignment.name = holdout.name; + assignment.iteration = holdout.iteration; + assignment.unitType = unitType; + assignment.eligible = true; + assignment.assigned = true; + assignment.variant = assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo); + + holdoutAssignmentCache_.put(holdout.id, new HoldoutAssignment(assignment, holdout, unitType)); + + return assignment; + } finally { + writeLock.unlock(); + } + } + private List getVariableExperiments(final String key) { return Concurrency.getRW(dataLock_, indexVariables_, key); } @@ -999,7 +1086,7 @@ private void clearRefreshTimer() { private static class ContextExperiment { Experiment data; - ExperimentHoldout[] holdouts; + Experiment[] holdouts; List> variables; Map customFieldValues; } @@ -1009,43 +1096,62 @@ private static class ContextCustomFieldValue { Object value; } - private static ExperimentHoldout[] resolveHoldouts(final int[] holdoutIds, - final Map holdoutIndex) { - if (holdoutIds == null || holdoutIds.length == 0) { + private static boolean isExcluded(final int[] excludedExperimentIds, final int experimentId) { + return (excludedExperimentIds != null) && (Arrays.binarySearch(excludedExperimentIds, experimentId) >= 0); + } + + // A holdout applies to an experiment when their unit types match and the experiment is not + // in the holdout's own exclusion list. A `full_on` holdout additionally applies only to + // experiments that are themselves full-on (fullOnVariant != 0); `full` holdouts apply + // regardless. This is derived once per experiment at data-install time, not per unit. + private static Experiment[] resolveApplicableHoldouts(final Experiment experiment, + final Map> holdoutsByUnitType) { + final List candidates = holdoutsByUnitType.get(experiment.unitType); + if (candidates == null || candidates.isEmpty()) { return null; } - final List resolved = new ArrayList(holdoutIds.length); - for (final int holdoutId : holdoutIds) { - final ExperimentHoldout holdout = holdoutIndex.get(holdoutId); - if (holdout != null && holdout.split != null && holdout.split.length > 0) { - resolved.add(holdout); + final List applicable = new ArrayList(candidates.size()); + for (final Experiment holdout : candidates) { + if ("full_on".equals(holdout.holdoutType) && experiment.fullOnVariant == 0) { + continue; + } + + if (isExcluded(holdout.excludedExperimentIds, experiment.id)) { + continue; } + + applicable.add(holdout); } - if (resolved.isEmpty()) { + if (applicable.isEmpty()) { return null; } - Collections.sort(resolved, new Comparator() { + Collections.sort(applicable, new Comparator() { @Override - public int compare(ExperimentHoldout a, ExperimentHoldout b) { + public int compare(Experiment a, Experiment b) { return Integer.valueOf(a.id).compareTo(b.id); } }); - return resolved.toArray(new ExperimentHoldout[0]); + return applicable.toArray(new Experiment[0]); } private void setData(final ContextData data) { final Map index = new HashMap(); final Map> indexVariables = new HashMap>(); - final Map holdoutIndex = new HashMap(); + final Map> holdoutsByUnitType = new HashMap>(); if (data.holdouts != null) { - for (final ExperimentHoldout holdout : data.holdouts) { - if (holdout != null) { - holdoutIndex.put(holdout.id, holdout); + for (final Experiment holdout : data.holdouts) { + if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { + List holdouts = holdoutsByUnitType.get(holdout.unitType); + if (holdouts == null) { + holdouts = new ArrayList(); + holdoutsByUnitType.put(holdout.unitType, holdouts); + } + holdouts.add(holdout); } } } @@ -1053,7 +1159,7 @@ private void setData(final ContextData data) { for (final Experiment experiment : data.experiments) { final ContextExperiment contextExperiment = new ContextExperiment(); contextExperiment.data = experiment; - contextExperiment.holdouts = resolveHoldouts(experiment.holdoutIds, holdoutIndex); + contextExperiment.holdouts = resolveApplicableHoldouts(experiment, holdoutsByUnitType); contextExperiment.variables = new ArrayList>(experiment.variants.length); for (final ExperimentVariant variant : experiment.variants) { @@ -1172,6 +1278,7 @@ private void logError(Throwable error) { private final Map hashedUnits_; private final Map assigners_; private final Map assignmentCache_ = new HashMap(); + private final Map holdoutAssignmentCache_ = new HashMap(); private final ReentrantLock eventLock_ = new ReentrantLock(); private final ArrayList exposures_ = new ArrayList(); From f0d99e03c03e7698628d187feba1ed3217b341bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 00:06:20 +0000 Subject: [PATCH 13/49] test: rewrite holdout contract tests for the first-class experiment model Replace the id-lookup ExperimentHoldout fixtures across ContextHoldoutTest, DefaultContextDataDeserializerTest, ContextDataTest, and holdouts_context.json with full Experiment entries carrying holdoutType and excludedExperimentIds. Cover suppression, control values, exclusion, the union rule across multiple holdouts, full vs full_on applicability, application-scoping independence, absent holdouts, custom-assignment precedence, and once-per-context holdout exposure emission, plus cross-SDK parity vectors for unicode unit ids, signed seed halves, and percentage/probability boundaries. Expected variants are computed from VariantAssigner rather than assumed. --- .../com/absmartly/sdk/ContextHoldoutTest.java | 750 +++++++++--------- .../DefaultContextDataDeserializerTest.java | 45 +- .../absmartly/sdk/json/ContextDataTest.java | 23 +- .../src/test/resources/holdouts_context.json | 56 +- 4 files changed, 478 insertions(+), 396 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 812f779..0eaecba 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -19,28 +19,35 @@ import com.absmartly.sdk.json.ContextData; import com.absmartly.sdk.json.Experiment; import com.absmartly.sdk.json.ExperimentApplication; -import com.absmartly.sdk.json.ExperimentHoldout; import com.absmartly.sdk.json.ExperimentVariant; import com.absmartly.sdk.json.Exposure; import com.absmartly.sdk.json.PublishEvent; import com.absmartly.sdk.json.Unit; +// A holdout arrives as an ordinary experiment entry (in ContextData.holdouts, not .experiments) so +// its variant semantics are: variant 0 = held out (no experimentation at all), variant 1 = exposed. +// Applicability to a covered experiment is derived client-side from unit type plus the full/full_on +// rule, minus that holdout's own excludedExperimentIds - there is no per-experiment holdoutIds field. +// A held-out unit gets control values and emits NO exposure for the experiments it covers; the +// holdout experiment itself always emits one ordinary exposure, cached once per context. class ContextHoldoutTest extends TestUtils { static final String UNIT_TYPE = "session_id"; static final String UID = "e791e240fcd3df7d238cfc285f475e8152fcc0ec"; + static final String UID_NOT_HELD_OUT = "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3"; - // split[0.5,0.5], seedHi=100, seedLo=200 -> variant 1 for UID above. + // split[0.5,0.5], seedHi=100, seedLo=200 -> variant 1 for UID, variant 0 for UID_NOT_HELD_OUT. static final int NORMAL_SEED_HI = 100; static final int NORMAL_SEED_LO = 200; static final int NORMAL_VARIANT = 1; - // split[0.1,0.9], seedHi=13, seedLo=111 -> variant 0 (i.e. user IS in this holdout) for UID above. - static final int HOLDOUT_IN_SEED_HI = 13; - static final int HOLDOUT_IN_SEED_LO = 111; + // split[0.1,0.9], seedHi=13, seedLo=111 -> variant 0 (held out) for UID, variant 1 for + // UID_NOT_HELD_OUT. + static final int HOLDOUT_A_SEED_HI = 13; + static final int HOLDOUT_A_SEED_LO = 111; - // split[0.1,0.9], seedHi=1, seedLo=222 -> variant 1 (i.e. user is NOT in this holdout) for UID above. - static final int HOLDOUT_OUT_SEED_HI = 1; - static final int HOLDOUT_OUT_SEED_LO = 222; + // split[0.1,0.9], seedHi=1, seedLo=222 -> variant 1 (not held out) for both UIDs. + static final int HOLDOUT_B_SEED_HI = 1; + static final int HOLDOUT_B_SEED_LO = 222; ContextDataProvider dataProvider; ContextEventLogger eventLogger; @@ -60,22 +67,34 @@ void setUp() { scheduler = mock(ScheduledExecutorService.class); } - Context createReadyContext(ContextData data) { - final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID); + Context createReadyContext(String uid, ContextData data) { + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, uid); return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, eventHandler, eventLogger, variableParser, audienceMatcher); } + Context createReadyContext(ContextData data) { + return createReadyContext(UID, data); + } + Context createReadyContext(ContextConfig config, ContextData data) { return Context.create(clock, config, scheduler, CompletableFuture.completedFuture(data), dataProvider, eventHandler, eventLogger, variableParser, audienceMatcher); } static Experiment newExperiment(int id, String name) { + return newExperiment(id, name, UNIT_TYPE, 0); + } + + static Experiment newExperiment(int id, String name, int fullOnVariant) { + return newExperiment(id, name, UNIT_TYPE, fullOnVariant); + } + + static Experiment newExperiment(int id, String name, String unitType, int fullOnVariant) { final Experiment experiment = new Experiment(); experiment.id = id; experiment.name = name; - experiment.unitType = UNIT_TYPE; + experiment.unitType = unitType; experiment.iteration = 1; experiment.seedHi = NORMAL_SEED_HI; experiment.seedLo = NORMAL_SEED_LO; @@ -83,255 +102,309 @@ static Experiment newExperiment(int id, String name) { experiment.trafficSeedHi = 1; experiment.trafficSeedLo = 2; experiment.trafficSplit = new double[]{0.0, 1.0}; - experiment.fullOnVariant = 0; + experiment.fullOnVariant = fullOnVariant; experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; experiment.variants = new ExperimentVariant[]{ new ExperimentVariant("A", null), - new ExperimentVariant("B", null) + fullOnVariant == 2 ? new ExperimentVariant("B", null) : new ExperimentVariant("B", null) }; experiment.audienceStrict = false; experiment.audience = null; return experiment; } - static ExperimentHoldout newHoldout(int id, int seedHi, int seedLo) { - return new ExperimentHoldout(id, seedHi, seedLo, new double[]{0.1, 0.9}); - } - - static ExperimentHoldout newFullOnHoldout(int id, int seedHi, int seedLo) { - return new ExperimentHoldout(id, seedHi, seedLo, new double[]{0.1, 0.9}, true); + static Experiment newHoldout(int id, String name, int seedHi, int seedLo) { + return newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "full", null); + } + + static Experiment newHoldout(int id, String name, String unitType, int seedHi, int seedLo, String holdoutType, + int[] excludedExperimentIds) { + final Experiment holdout = new Experiment(); + holdout.id = id; + holdout.name = name; + holdout.unitType = unitType; + holdout.iteration = 1; + holdout.seedHi = seedHi; + holdout.seedLo = seedLo; + holdout.split = new double[]{0.1, 0.9}; + holdout.trafficSplit = new double[]{0.0, 1.0}; + holdout.fullOnVariant = 0; + holdout.applications = new ExperimentApplication[0]; + holdout.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + holdout.audienceStrict = false; + holdout.audience = null; + holdout.holdoutType = holdoutType; + holdout.excludedExperimentIds = excludedExperimentIds; + return holdout; } static ContextData contextDataOf(Experiment... experiments) { - return contextDataOf(new ExperimentHoldout[0], experiments); + return contextDataOf(null, experiments); } - static ContextData contextDataOf(ExperimentHoldout[] holdouts, Experiment... experiments) { + static ContextData contextDataOf(Experiment[] holdouts, Experiment... experiments) { final ContextData data = new ContextData(); data.experiments = experiments; data.holdouts = holdouts; return data; } - @Test - void assignsNormallyWhenExperimentHasNoHoldouts() { - final Experiment experiment = newExperiment(1, "exp_no_holdout"); + PublishEvent publishedEvent(String uid, Exposure... exposures) { + return publishedEvent(UNIT_TYPE, uid, exposures); + } - final Context context = createReadyContext(contextDataOf(experiment)); + PublishEvent publishedEvent(String unitType, String uid, Exposure... exposures) { + final PublishEvent expected = new PublishEvent(); + expected.hashed = true; + expected.publishedAt = clock.millis(); + expected.units = new Unit[]{ + new Unit(unitType, new String(Hashing.hashUnit(uid), StandardCharsets.US_ASCII)) + }; + expected.exposures = exposures; + return expected; + } - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_holdout")); + Exposure holdoutExposure(int id, String name, int variant) { + return holdoutExposure(UNIT_TYPE, id, name, variant); } + Exposure holdoutExposure(String unitType, int id, String name, int variant) { + return new Exposure(id, name, unitType, variant, clock.millis(), true, true, false, false, false, false); + } + + // (1) + (2): a held-out unit gets control values and emits zero exposures for the experiment + // it is held out of. @Test - void assignsControlVariantWhenUnitIsInHoldout() { + void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { final Experiment experiment = newExperiment(1, "exp_holdout_in"); - experiment.holdoutIds = new int[]{11}; - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); assertEquals(0, context.peekTreatment("exp_holdout_in")); + + context.getTreatment("exp_holdout_in"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // (3): the holdout's own exposure is exactly one ordinary exposure with the unit's holdout + // variant and no holdout-specific fields, distinct from Exposure() itself no longer having + // heldOut/holdoutId fields at all. @Test - void assignsNormallyWhenUnitIsNotInHoldout() { - final Experiment experiment = newExperiment(1, "exp_holdout_out"); - experiment.holdoutIds = new int[]{11}; - + void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { + final Experiment experiment = newExperiment(1, "exp_holdout_in"); final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + context.getTreatment("exp_holdout_in"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_holdout_out")); + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // (4): a non-held-out unit emits the holdout exposure with variant=1 plus its own normal + // exposure, unchanged. @Test - void checksAllHoldoutsUntilAMatchIsFound() { - final Experiment experiment = newExperiment(1, "exp_multi_holdout"); - experiment.holdoutIds = new int[]{11, 12}; + void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { + final Experiment experiment = newExperiment(1, "exp_holdout_out"); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{ - newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), - newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - }, experiment)); + assertEquals(0, context.getTreatment("exp_holdout_out")); // UID_NOT_HELD_OUT's normal variant is 0 + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - assertEquals(0, context.peekTreatment("exp_multi_holdout")); + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_holdout_out", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, + false), + holdoutExposure(11, "holdout_a", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // (5): an experiment excluded from a holdout is never covered by it and emits normally for + // both a held-out and a non-held-out unit, even while the same unit is suppressed elsewhere. @Test - void evaluatesMultipleHoldoutsInCanonicalIdOrder() { - final Experiment experiment = newExperiment(1, "exp_ordered_holdouts"); - experiment.holdoutIds = new int[]{42, 11}; + void excludedExperimentEmitsNormallyEvenForHeldOutUnit() { + final Experiment covered = newExperiment(1, "exp_holdout_in"); + final Experiment excluded = newExperiment(2, "exp_excluded"); + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full", + new int[]{2}); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{ - newHoldout(42, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - }, experiment)); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); - assertEquals(0, context.getTreatment("exp_ordered_holdouts")); + assertEquals(0, context.getTreatment("exp_holdout_in")); // suppressed -> control + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_excluded")); // exclusion -> unaffected when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); context.publish(); - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_ordered_holdouts", UNIT_TYPE, 0, clock.millis(), true, true, false, false, - false, false, true, 11), - }; - + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + new Exposure(2, "exp_excluded", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, + false, false)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // (6): a normal experiment covered by two holdouts is suppressed if the unit is held out by + // EITHER one (union), and each applicable holdout still emits its own independent exposure. @Test - void assignsNormallyWhenNotInAnyHoldout() { - final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); - experiment.holdoutIds = new int[]{11, 12}; - + void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { + final Experiment experiment = newExperiment(1, "exp_multi_holdout"); final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{ - newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), - newHoldout(12, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO), + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out }, experiment)); - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_multi_holdout_miss")); + assertEquals(0, context.getTreatment("exp_multi_holdout")); // union -> suppressed + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + holdoutExposure(12, "holdout_b", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } @Test - void sharesHoldoutDefinitionsAcrossExperiments() { - final Experiment experimentA = newExperiment(1, "exp_shared_a"); - experimentA.holdoutIds = new int[]{11}; - final Experiment experimentB = newExperiment(2, "exp_shared_b"); - experimentB.holdoutIds = new int[]{11}; + void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { + final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), + }, experiment)); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, - experimentA, experimentB)); + assertEquals(0, context.getTreatment("exp_multi_holdout_miss")); // UID_NOT_HELD_OUT's normal variant + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - // same seed -> same membership across both experiments referencing the shared holdout - assertEquals(0, context.peekTreatment("exp_shared_a")); - assertEquals(0, context.peekTreatment("exp_shared_b")); + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_multi_holdout_miss", UNIT_TYPE, 0, clock.millis(), true, true, false, false, + false, false), + holdoutExposure(11, "holdout_a", 1), + holdoutExposure(12, "holdout_b", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // (7): a full_on holdout is skipped entirely for a non-full-on experiment (no suppression, no + // exposure triggered), and applies normally to a full-on one; a `full` holdout applies to a + // full-on experiment regardless of its fullOnVariant. @Test - void assignsNormallyWhenHoldoutIdIsUnknown() { - final Experiment experiment = newExperiment(1, "exp_unknown_holdout"); - experiment.holdoutIds = new int[]{99}; // no matching top-level definition - + void fullOnHoldoutSkippedForNonFullOnExperiment() { + final Experiment experiment = newExperiment(1, "exp_regular"); final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_unknown_holdout")); - } + new Experiment[]{newHoldout(11, "holdout_fullon", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full_on", null)}, + experiment)); - @Test - void resolvesKnownHoldoutAndSkipsUnknownId() { - final Experiment experiment = newExperiment(1, "exp_mixed_holdout"); - experiment.holdoutIds = new int[]{99, 11}; // 99 is unknown, 11 resolves and holds the unit out + // the unit would be held out (matches the holdout's split), but full_on holdouts only + // cover full-on experiments, so evaluation must proceed normally. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_regular")); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - assertEquals(0, context.peekTreatment("exp_mixed_holdout")); + // the full_on holdout must not fire either - it never became applicable to any evaluated + // experiment in this context. + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_regular", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, + false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } @Test - void skipsMalformedHoldoutWithNullSplit() { - final Experiment experiment = newExperiment(1, "exp_null_split_holdout"); - experiment.holdoutIds = new int[]{11}; - - // a holdout whose split is missing from the payload must be dropped, not crash assignment + void fullOnHoldoutAppliesToFullOnExperiment() { + final Experiment experiment = newExperiment(1, "exp_fullon", 2); final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{new ExperimentHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO, null)}, + new Experiment[]{newHoldout(11, "holdout_fullon", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full_on", null)}, experiment)); - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_null_split_holdout")); + assertEquals(0, context.getTreatment("exp_fullon")); // suppressed -> control + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_fullon", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } @Test - void skipsHoldoutWhenUnitMissingForUnitType() { - final Experiment experiment = newExperiment(1, "exp_no_unit_holdout"); - experiment.unitType = "user_id"; // no unit registered for this type - experiment.holdoutIds = new int[]{11}; - + void fullHoldoutAppliesToFullOnExperimentRegardlessOfFullOnVariant() { + final Experiment experiment = newExperiment(1, "exp_fullon_full_holdout", 2); final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - // holdout evaluation is skipped (no uid); assignment falls through unassigned -> control 0, not held out - assertEquals(0, context.getTreatment("exp_no_unit_holdout")); + assertEquals(0, context.getTreatment("exp_fullon_full_holdout")); // held out despite fullOnVariant=2 + } - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); + // (8): holdout applicability is derived from unit type alone; the `applications` field (which + // the wire contract leaves empty on holdout entries) plays no role in matching. + @Test + void applicabilityIgnoresApplicationsFieldOnBothSides() { + final Experiment experiment = newExperiment(1, "exp_scoped"); + experiment.applications = new ExperimentApplication[]{new ExperimentApplication("mobile")}; - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_no_unit_holdout", "user_id", 0, clock.millis(), false, true, false, false, false, - false, false, 0), - }; + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + holdout.applications = null; // matches the wire contract's empty applications array - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(0, context.peekTreatment("exp_scoped")); // still held out despite mismatched applications } + // A holdout only covers experiments sharing its unit type; a matching split/seed on a + // different unit type must never suppress. @Test - void reusesCachedAssignmentForHeldOutExperiment() { - final Experiment experiment = newExperiment(1, "exp_holdout_cache"); - experiment.holdoutIds = new int[]{11}; + void holdoutDoesNotApplyToDifferentUnitType() { + final Experiment experiment = newExperiment(1, "exp_session", UNIT_TYPE, 0); + final Experiment holdout = newHoldout(11, "holdout_user", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", null); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); - assertEquals(0, context.getTreatment("exp_holdout_cache")); - // second call hits the cache (holdouts present and unchanged) -> no new exposure - assertEquals(0, context.getTreatment("exp_holdout_cache")); - assertEquals(1, context.getPendingCount()); + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_session")); } + // (9): an absent holdouts key (null, the ContextData default) leaves behaviour identical to + // pre-holdout: no suppression, no NPE. @Test - void holdoutTakesPrecedenceOverAudienceMismatch() { - final Experiment experiment = newExperiment(1, "exp_holdout_audience"); - experiment.audienceStrict = true; - experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; - experiment.holdoutIds = new int[]{11}; - - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - context.setAttribute("age", 5); // would mismatch the audience filter if evaluated + void assignsNormallyWhenHoldoutsKeyIsAbsent() { + final Experiment experiment = newExperiment(1, "exp_no_holdouts_key"); + final Context context = createReadyContext(contextDataOf(experiment)); - assertEquals(0, context.peekTreatment("exp_holdout_audience")); + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_holdouts_key")); } @Test - void overrideTakesPrecedenceOverHoldout() { - final Experiment experiment = newExperiment(1, "exp_holdout_override"); - experiment.holdoutIds = new int[]{11}; + void assignsNormallyWhenExperimentUnitTypeHasNoHoldouts() { + final Experiment experiment = newExperiment(1, "exp_no_matching_holdouts"); + final Experiment holdout = newHoldout(11, "holdout_other", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", null); - final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setOverride( - "exp_holdout_override", 3); - final Context context = createReadyContext(config, contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); - assertEquals(3, context.peekTreatment("exp_holdout_override")); + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_matching_holdouts")); } + // (10): a custom assignment can never override a held-out unit's variant - holdout precedence + // beats custom assignments, matching the existing audience/full-on/traffic precedence rules. @Test - void holdoutTakesPrecedenceOverCustomAssignment() { + void customAssignmentCannotOverrideHeldOutVariant() { final Experiment experiment = newExperiment(1, "exp_holdout_custom"); - experiment.holdoutIds = new int[]{11}; final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( "exp_holdout_custom", 3); final Context context = createReadyContext(config, contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); assertEquals(0, context.peekTreatment("exp_holdout_custom")); } @@ -339,303 +412,232 @@ void holdoutTakesPrecedenceOverCustomAssignment() { @Test void reusesCachedHeldOutAssignmentWithCustomAssignment() { final Experiment experiment = newExperiment(1, "exp_holdout_custom_cache"); - experiment.holdoutIds = new int[]{11}; final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( "exp_holdout_custom_cache", 3); final Context context = createReadyContext(config, contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); // the custom assignment can never apply while held out; repeated calls must hit the - // cache and not queue duplicate exposures + // cache and not queue duplicate exposures. assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); assertEquals(0, context.getTreatment("exp_holdout_custom_cache")); - assertEquals(1, context.getPendingCount()); + assertEquals(1, context.getPendingCount()); // one exposure: the holdout's own } @Test - void reusesCachedAudienceMismatchAssignmentWithCustomAssignment() { - final Experiment experiment = newExperiment(1, "exp_audience_custom_cache"); - experiment.audienceStrict = true; - experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + void overrideTakesPrecedenceOverHoldout() { + final Experiment experiment = newExperiment(1, "exp_holdout_override"); - final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( - "exp_audience_custom_cache", 3); - final Context context = createReadyContext(config, contextDataOf(experiment)); - context.setAttribute("age", 5); // mismatches the strict audience -> variant forced to 0 + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setOverride( + "exp_holdout_override", 3); + final Context context = createReadyContext(config, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - // strict audience mismatch forces variant 0; the custom assignment can never apply, so - // repeated calls must hit the cache and not queue duplicate exposures - assertEquals(0, context.getTreatment("exp_audience_custom_cache")); - assertEquals(0, context.getTreatment("exp_audience_custom_cache")); - assertEquals(1, context.getPendingCount()); + assertEquals(3, context.peekTreatment("exp_holdout_override")); } @Test - void reusesCachedTrafficIneligibleAssignmentWithCustomAssignment() { - final Experiment experiment = newExperiment(1, "exp_traffic_custom_cache"); - experiment.trafficSplit = new double[]{1.0, 0.0}; // unit is NOT in the experiment traffic + void holdoutTakesPrecedenceOverAudienceMismatch() { + final Experiment experiment = newExperiment(1, "exp_holdout_audience"); + experiment.audienceStrict = true; + experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; - final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( - "exp_traffic_custom_cache", 3); - final Context context = createReadyContext(config, contextDataOf(experiment)); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + context.setAttribute("age", 5); // would mismatch the audience filter if evaluated - // traffic ineligibility forces variant 0; the custom assignment can never apply, so repeated - // calls must hit the cache and not queue duplicate exposures - assertEquals(0, context.getTreatment("exp_traffic_custom_cache")); - assertEquals(0, context.getTreatment("exp_traffic_custom_cache")); - assertEquals(1, context.getPendingCount()); + assertEquals(0, context.peekTreatment("exp_holdout_audience")); } + // (11): the holdout's own exposure is emitted once per context, not once per suppressed + // experiment - two experiments covered by the same holdout still yield a single holdout + // exposure. @Test - void holdoutTakesPrecedenceOverFullOn() { - final Experiment experiment = newExperiment(1, "exp_holdout_fullon"); - experiment.fullOnVariant = 2; - experiment.holdoutIds = new int[]{11}; + void holdoutExposureEmittedOncePerContextNotPerSuppressedExperiment() { + final Experiment experimentA = newExperiment(1, "exp_shared_a"); + final Experiment experimentB = newExperiment(2, "exp_shared_b"); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experimentA, + experimentB)); - assertEquals(0, context.peekTreatment("exp_holdout_fullon")); + assertEquals(0, context.getTreatment("exp_shared_a")); + assertEquals(0, context.getTreatment("exp_shared_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } @Test - void refreshReassignsWhenHoldoutsChange() { - final Experiment experiment = newExperiment(1, "exp_holdout_refresh"); + void holdoutExposureFiresOnceEvenWhenTriggeringExperimentIsNotSuppressed() { + final Experiment experimentA = newExperiment(1, "exp_shared_a"); + final Experiment experimentB = newExperiment(2, "exp_shared_b"); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - final Context context = createReadyContext(contextDataOf(experiment)); + final Context context = createReadyContext(UID_NOT_HELD_OUT, + contextDataOf(new Experiment[]{holdout}, experimentA, experimentB)); - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_refresh")); - assertEquals(1, context.getPendingCount()); + context.getTreatment("exp_shared_a"); + context.getTreatment("exp_shared_b"); - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_refresh"); - refreshedExperiment.holdoutIds = new int[]{11}; + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - final CompletableFuture refreshFuture = new CompletableFuture<>(); - when(dataProvider.getContextData()).thenReturn(refreshFuture); + final PublishEvent expected = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_shared_a", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, false), + holdoutExposure(11, "holdout_a", 1), + new Exposure(2, "exp_shared_b", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } - final CompletableFuture refreshing = context.refreshAsync(); - refreshFuture.complete(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, refreshedExperiment)); - refreshing.join(); + // Refresh with an unchanged holdout definition must not re-trigger the holdout's exposure or + // change the covered experiment's cached suppression. + @Test + void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { + final Experiment experiment = newExperiment(1, "exp_holdout_cache"); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - assertEquals(0, context.getTreatment("exp_holdout_refresh")); - assertEquals(2, context.getPendingCount()); // holdout change triggered a new exposure + assertEquals(0, context.getTreatment("exp_holdout_cache")); + assertEquals(0, context.getTreatment("exp_holdout_cache")); + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure } @Test - void refreshReassignsWhenReferencedHoldoutDefinitionChanges() { - final Experiment experiment = newExperiment(1, "exp_holdout_def_change"); - experiment.holdoutIds = new int[]{11}; + void refreshReassignsWhenHoldoutDefinitionChanges() { + final Experiment experiment = newExperiment(1, "exp_holdout_refresh"); - // unit is NOT in the holdout initially + // unit is NOT held out initially (holdout B's seed) final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, experiment)); - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_def_change")); - assertEquals(1, context.getPendingCount()); - - // same holdoutIds, but the referenced definition's seed changes so the unit is now IN the holdout - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_def_change"); - refreshedExperiment.holdoutIds = new int[]{11}; + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_refresh")); + assertEquals(2, context.getPendingCount()); // normal exposure + holdout exposure (variant 1) + // same holdout id, but the definition's seed changes so the unit is now held out + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_refresh"); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); final CompletableFuture refreshing = context.refreshAsync(); refreshFuture.complete(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, refreshedExperiment)); + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + refreshedExperiment)); refreshing.join(); - assertEquals(0, context.getTreatment("exp_holdout_def_change")); - assertEquals(2, context.getPendingCount()); // changed definition triggered a new exposure + assertEquals(0, context.getTreatment("exp_holdout_refresh")); + // refreshed holdout re-exposes (its definition changed); the now-suppressed experiment + // emits no exposure of its own. + assertEquals(3, context.getPendingCount()); } - @Test - void exposureCarriesHeldOutAndHoldoutId() { - final Experiment experiment = newExperiment(1, "exp_holdout_exposure"); - experiment.holdoutIds = new int[]{42}; - - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(42, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - - assertEquals(0, context.getTreatment("exp_holdout_exposure")); - - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + // --- Cross-SDK parity vectors ----------------------------------------------------------- + // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> + // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the + // collector's server-side verdict for the same fixtures (test_holdouts.py :: + // TestCollectorHoldoutVerdictVectors), pinning unicode unit ids, a seed whose high AND low + // 32-bit halves both have the sign bit set, a 1% percentage boundary, an assignment-probability + // boundary immediately either side of 10%, and a unit held out by two holdouts simultaneously. - context.publish(); + static final String VERDICT_UNIT_TYPE = "verdict_unit_type"; - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_holdout_exposure", UNIT_TYPE, 0, clock.millis(), true, true, false, false, false, - false, true, 42), - }; - - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + Context verdictContext(String uid, Experiment... holdouts) { + final Experiment experiment = newExperiment(9, "verdict_vectors_experiment", VERDICT_UNIT_TYPE, 0); + final ContextConfig config = ContextConfig.create().setUnit(VERDICT_UNIT_TYPE, uid); + return createReadyContext(config, contextDataOf(holdouts, experiment)); } - @Test - void exposureCarriesNotHeldOutFieldsWhenUnitNotInHoldout() { - final Experiment experiment = newExperiment(1, "exp_holdout_out_exposure"); - experiment.holdoutIds = new int[]{11}; - - // holdout evaluated but unit is NOT in it -> normal assignment, heldOut=false, holdoutId=0 - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_OUT_SEED_HI, HOLDOUT_OUT_SEED_LO)}, experiment)); - - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_out_exposure")); + static Experiment verdictHoldout(int id, int seedHi, int seedLo, double[] split) { + final Experiment holdout = newHoldout(id, "holdout_" + id, VERDICT_UNIT_TYPE, seedHi, seedLo, "full", null); + holdout.split = split; + return holdout; + } + // Verdict is read off the holdout's own exposure variant (0 = held out), the exact quantity + // the collector's TestCollectorHoldoutVerdictVectors checks. Suppression is exercised too: a + // held-out unit's covered-experiment variant must be forced to control (0) and emit no + // exposure of its own, regardless of that experiment's independent seed. + void assertHeldOutBy(Context context, int holdoutId, String holdoutName) { + assertEquals(0, context.getTreatment("verdict_vectors_experiment")); when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_holdout_out_exposure", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, - false, - false, false, false, false, 0), - }; - - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); - } - - @Test - void assignsNormallyWhenHoldoutIdsIsEmpty() { - final Experiment experiment = newExperiment(1, "exp_empty_holdout_ids"); - experiment.holdoutIds = new int[0]; - - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_empty_holdout_ids")); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(org.mockito.ArgumentMatchers.eq(context), + org.mockito.ArgumentMatchers.argThat(event -> (event.exposures.length == 1) + && (event.exposures[0].id == holdoutId) && event.exposures[0].name.equals(holdoutName) + && (event.exposures[0].variant == 0))); } - @Test - void skipsHoldoutWithEmptySplit() { - final Experiment experiment = newExperiment(1, "exp_empty_split_holdout"); - experiment.holdoutIds = new int[]{11}; - - // a holdout with an empty split must be dropped, not hold the unit out - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{new ExperimentHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO, - new double[0])}, - experiment)); + void assertNotHeldOut(Context context, int holdoutId, String holdoutName) { + context.getTreatment("verdict_vectors_experiment"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_empty_split_holdout")); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(org.mockito.ArgumentMatchers.eq(context), + org.mockito.ArgumentMatchers.argThat(event -> java.util.Arrays.stream(event.exposures) + .anyMatch(e -> (e.id == holdoutId) && e.name.equals(holdoutName) && (e.variant == 1)))); } @Test - void fullOnHoldoutAppliesToFullOnExperiment() { - final Experiment experiment = newExperiment(1, "exp_fullon_holdout_fullon_exp"); - experiment.fullOnVariant = 2; - experiment.holdoutIds = new int[]{11}; + void unicodeUnitIds() { + // BMP diacritics, held out by a holdout with seed=42, 10%. + assertHeldOutBy(verdictContext("façade", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, + "holdout_107"); - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - - assertEquals(0, context.peekTreatment("exp_fullon_holdout_fullon_exp")); + // CJK + emoji combination, not held out by the same holdout. + assertNotHeldOut(verdictContext("unicode_emoji_\uD83D\uDE80_0", + verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, "holdout_107"); } @Test - void fullOnHoldoutIsSkippedForRegularExperiment() { - final Experiment experiment = newExperiment(1, "exp_fullon_holdout_regular_exp"); - experiment.holdoutIds = new int[]{11}; - - // the unit would be held out (matches the holdout's split), but fullOn holdouts only - // apply to full-on experiments (fullOnVariant != 0), so it must be skipped entirely and - // normal assignment must proceed. - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); + void signedSeedHalves() { + // seed = -9223372034707292160 (0x8000000080000000 as signed int64) has the sign bit set + // in BOTH seedHi and seedLo once split via (seed >> 32) / (int) seed. + final int seedHi = (int) (-9223372034707292160L >> 32); + final int seedLo = (int) -9223372034707292160L; - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_fullon_holdout_regular_exp")); + assertHeldOutBy(verdictContext("signed_unit_4", verdictHoldout(106, seedHi, seedLo, new double[]{0.1, 0.9})), + 106, "holdout_106"); + assertNotHeldOut(verdictContext("signed_unit_b_1", + verdictHoldout(106, seedHi, seedLo, new double[]{0.1, 0.9})), 106, "holdout_106"); } @Test - void absentFullOnBehavesAsFullHoldoutOnFullOnExperiment() { - final Experiment experiment = newExperiment(1, "exp_absent_fullon_holdout_fullon_exp"); - experiment.fullOnVariant = 2; - experiment.holdoutIds = new int[]{11}; - - // fullOn absent (null) -> treated as a regular (full) holdout, applies regardless of - // whether the experiment is full-on. - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{newHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO)}, experiment)); - - assertEquals(0, context.peekTreatment("exp_absent_fullon_holdout_fullon_exp")); + void percentageBoundaryAtOnePercent() { + assertHeldOutBy(verdictContext("pct1_unit_188", verdictHoldout(105, 0, 999999, new double[]{0.01, 0.99})), + 105, "holdout_105"); + assertNotHeldOut(verdictContext("pct1_unit_0", verdictHoldout(105, 0, 999999, new double[]{0.01, 0.99})), + 105, "holdout_105"); } @Test - void mixedFullAndFullOnHoldoutsOnRegularExperiment() { - final Experiment experiment = newExperiment(1, "exp_mixed_full_fullon_regular"); - experiment.holdoutIds = new int[]{11, 12}; - - // 11 is a full_on holdout that would match but must be skipped (regular experiment); - // 12 is a regular (full) holdout that matches -> unit is held out via 12. - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{ - newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - }, experiment)); - - assertEquals(0, context.getTreatment("exp_mixed_full_fullon_regular")); - - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); - - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_mixed_full_fullon_regular", UNIT_TYPE, 0, clock.millis(), true, true, false, - false, false, false, true, 12), - }; - - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + void assignmentProbabilityBoundaryAroundTenPercent() { + // closest computed assignment probabilities immediately below/above the 10% threshold for + // seed=42, pinning the exact split boundary behaviour (prob < cumSum). + assertHeldOutBy( + verdictContext("boundary_unit_175653", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), 107, + "holdout_107"); + assertNotHeldOut(verdictContext("boundary_unit_75792", verdictHoldout(107, 0, 42, new double[]{0.1, 0.9})), + 107, "holdout_107"); } @Test - void mixedFullAndFullOnHoldoutsOnFullOnExperiment() { - final Experiment experiment = newExperiment(1, "exp_mixed_full_fullon_fullon_exp"); - experiment.fullOnVariant = 2; - experiment.holdoutIds = new int[]{11, 12}; - - // on a full-on experiment both holdouts apply; iteration order is by payload id, so the - // full_on holdout (11) matches first. - final Context context = createReadyContext(contextDataOf( - new ExperimentHoldout[]{ - newFullOnHoldout(11, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - newHoldout(12, HOLDOUT_IN_SEED_HI, HOLDOUT_IN_SEED_LO), - }, experiment)); - - assertEquals(0, context.getTreatment("exp_mixed_full_fullon_fullon_exp")); + void unitHeldOutByTwoHoldoutsSimultaneously() { + final Context context = verdictContext("dual_holdout_unit_5", + verdictHoldout(107, 0, 42, new double[]{0.1, 0.9}), + verdictHoldout(108, 0, 55, new double[]{0.1, 0.9})); + assertEquals(0, context.getTreatment("verdict_vectors_experiment")); when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); context.publish(); - final PublishEvent expected = new PublishEvent(); - expected.hashed = true; - expected.publishedAt = clock.millis(); - expected.units = new Unit[]{ - new Unit(UNIT_TYPE, new String(Hashing.hashUnit(UID), StandardCharsets.US_ASCII)) - }; - expected.exposures = new Exposure[]{ - new Exposure(1, "exp_mixed_full_fullon_fullon_exp", UNIT_TYPE, 0, clock.millis(), true, true, false, - false, false, false, true, 11), - }; - + final PublishEvent expected = publishedEvent(VERDICT_UNIT_TYPE, "dual_holdout_unit_5", + holdoutExposure(VERDICT_UNIT_TYPE, 107, "holdout_107", 0), + holdoutExposure(VERDICT_UNIT_TYPE, 108, "holdout_108", 0)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } } diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index 7471b20..263981c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -7,7 +7,6 @@ import com.absmartly.sdk.json.ContextData; import com.absmartly.sdk.json.Experiment; import com.absmartly.sdk.json.ExperimentApplication; -import com.absmartly.sdk.json.ExperimentHoldout; import com.absmartly.sdk.json.ExperimentVariant; class DefaultContextDataDeserializerTest extends TestUtils { @@ -151,14 +150,48 @@ void deserializeHoldouts() { }; experiment.audienceStrict = false; experiment.audience = null; - experiment.holdoutIds = new int[]{11, 12}; + + final Experiment holdoutA = new Experiment(); + holdoutA.id = 11; + holdoutA.name = "holdout_a"; + holdoutA.unitType = "session_id"; + holdoutA.iteration = 1; + holdoutA.seedHi = 13; + holdoutA.seedLo = 111; + holdoutA.split = new double[]{0.1, 0.9}; + holdoutA.trafficSplit = new double[]{0.0, 1.0}; + holdoutA.fullOnVariant = 0; + holdoutA.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + holdoutA.audienceStrict = false; + holdoutA.audience = null; + holdoutA.holdoutType = "full"; + holdoutA.excludedExperimentIds = new int[0]; + + final Experiment holdoutB = new Experiment(); + holdoutB.id = 12; + holdoutB.name = "holdout_b"; + holdoutB.unitType = "session_id"; + holdoutB.iteration = 1; + holdoutB.seedHi = 1; + holdoutB.seedLo = 222; + holdoutB.split = new double[]{0.05, 0.95}; + holdoutB.trafficSplit = new double[]{0.0, 1.0}; + holdoutB.fullOnVariant = 0; + holdoutB.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null) + }; + holdoutB.audienceStrict = false; + holdoutB.audience = null; + holdoutB.holdoutType = "full_on"; + holdoutB.excludedExperimentIds = new int[]{4}; final ContextData expected = new ContextData( new Experiment[]{experiment}, - new ExperimentHoldout[]{ - new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}), - new ExperimentHoldout(12, 1, 222, new double[]{0.05, 0.95}) - }); + new Experiment[]{holdoutA, holdoutB}); assertNotNull(data); assertEquals(expected, data); diff --git a/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java index c798cdc..a939572 100644 --- a/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/json/ContextDataTest.java @@ -16,26 +16,31 @@ private static Experiment experiment(int id, String name) { return experiment; } + private static Experiment holdout(int id, String unitType) { + final Experiment holdout = new Experiment(); + holdout.id = id; + holdout.name = "holdout_" + id; + holdout.unitType = unitType; + holdout.split = new double[]{0.1, 0.9}; + holdout.variants = new ExperimentVariant[0]; + holdout.holdoutType = "full"; + return holdout; + } + @Test void equalsHashCodeAndToStringWithHoldouts() { final Experiment[] experiments = new Experiment[]{experiment(1, "exp")}; - final ExperimentHoldout[] holdouts = new ExperimentHoldout[]{ - new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}) - }; + final Experiment[] holdouts = new Experiment[]{holdout(11, "session_id")}; final ContextData a = new ContextData(experiments, holdouts); - final ContextData b = new ContextData(experiments, new ExperimentHoldout[]{ - new ExperimentHoldout(11, 13, 111, new double[]{0.1, 0.9}) - }); + final ContextData b = new ContextData(experiments, new Experiment[]{holdout(11, "session_id")}); assertEquals(a, a); assertEquals(a, b); assertEquals(a.hashCode(), b.hashCode()); // same experiments but different holdouts must compare unequal - final ContextData differentHoldouts = new ContextData(experiments, new ExperimentHoldout[]{ - new ExperimentHoldout(22, 13, 111, new double[]{0.1, 0.9}) - }); + final ContextData differentHoldouts = new ContextData(experiments, new Experiment[]{holdout(22, "session_id")}); assertNotEquals(a, differentHoldouts); assertNotEquals(a.hashCode(), differentHoldouts.hashCode()); diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json index dd64e47..b51da45 100644 --- a/core-api/src/test/resources/holdouts_context.json +++ b/core-api/src/test/resources/holdouts_context.json @@ -33,31 +33,73 @@ "config":"{\"banner.border\":1,\"banner.size\":\"large\"}" } ], - "audience": null, - "holdoutIds":[ - 11, - 12 - ] + "audience": null } ], "holdouts":[ { "id":11, + "name":"holdout_a", + "iteration":1, + "unitType":"session_id", "seedHi":13, "seedLo":111, "split":[ 0.1, 0.9 - ] + ], + "trafficSeedHi":0, + "trafficSeedLo":0, + "trafficSplit":[ + 0.0, + 1.0 + ], + "fullOnVariant":0, + "variants":[ + { + "name":"A", + "config":null + }, + { + "name":"B", + "config":null + } + ], + "audience": null, + "holdoutType":"full", + "excludedExperimentIds":[] }, { "id":12, + "name":"holdout_b", + "iteration":1, + "unitType":"session_id", "seedHi":1, "seedLo":222, "split":[ 0.05, 0.95 - ] + ], + "trafficSeedHi":0, + "trafficSeedLo":0, + "trafficSplit":[ + 0.0, + 1.0 + ], + "fullOnVariant":0, + "variants":[ + { + "name":"A", + "config":null + }, + { + "name":"B", + "config":null + } + ], + "audience": null, + "holdoutType":"full_on", + "excludedExperimentIds":[4] } ] } From 6c3a4f4feea3b6a299dec259e399e90bdc169969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 12:38:07 +0000 Subject: [PATCH 14/49] fix: pin holdout cache identity to (id, iteration) to stop exposure thrash The holdout Assignment cache and experimentMatches both invalidated on fields that never change who is a member: HoldoutAssignment.matches compared seedHi/seedLo/split, and experimentMatches compared applicable holdouts with Arrays.equals over Experiment (name, variants, applications, audience, customFieldValues included). Either kind of edit replaced the cached Assignment, reset its once-per-context exposed flag, and let an already-exposed unit be re-assigned into the other arm of the same holdout - violating the invariant that a unit must never appear in both arms of a holdout within one context's lifetime. Both invalidation paths (HoldoutAssignment.matches and experimentMatches's holdout comparison) and the cache key/replacement path in getHoldoutAssignment now agree on the same identity: a holdout's (id, iteration) pair, mirroring the granularity experimentMatches already used for ordinary experiments. Only an iteration bump - a genuine re-randomization epoch - replaces a cached verdict; seed, split, and any cosmetic field can change freely without disturbing an already-recorded arm. --- .../main/java/com/absmartly/sdk/Context.java | 70 +++++++--- .../com/absmartly/sdk/ContextHoldoutTest.java | 128 ++++++++++++++++-- 2 files changed, 166 insertions(+), 32 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 9f2ebcb..fe309df 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -721,7 +721,31 @@ private boolean experimentMatches(final ContextExperiment experiment, final Assi experiment.data.iteration == assignment.iteration && experiment.data.fullOnVariant == assignment.fullOnVariant && Arrays.equals(experiment.data.trafficSplit, assignment.trafficSplit) && - Arrays.equals(experiment.holdouts, assignment.holdouts); + holdoutSetMatches(experiment.holdouts, assignment.holdouts); + } + + // Applicable-holdout identity for cache-validity purposes is (id, iteration) per entry, in + // resolution order - the same fields experimentMatches already uses to identify an ordinary + // experiment's own run, and nothing finer. Seed, split, name, variants, applications, audience + // and customFieldValues can all change without altering who is covered or which arm a unit + // lands in, so comparing whole Experiment objects (Arrays.equals delegates to + // Experiment.equals) would invalidate on purely cosmetic edits and force a duplicate exposure. + // A change in the resolved set - membership added/removed, or an id/iteration change on an + // existing entry - does alter coverage and must invalidate. + private static boolean holdoutSetMatches(final Experiment[] a, final Experiment[] b) { + if (a == b) { + return true; + } else if ((a == null) || (b == null) || (a.length != b.length)) { + return false; + } + + for (int i = 0; i < a.length; ++i) { + if ((a[i].id != b[i].id) || (a[i].iteration != b[i].iteration)) { + return false; + } + } + + return true; } // A custom assignment can only take effect on the normal, traffic-eligible assignment path. @@ -754,9 +778,10 @@ private static class Assignment { boolean audienceMismatch; // Held out by a union of applicable holdouts: no exposure for this experiment, control - // values only. `holdouts` is the resolved applicable list, used both to invalidate this - // cached assignment when a holdout definition changes and to trigger each holdout's own - // exposure once this experiment is evaluated (see queueExposure). + // values only. `holdouts` is the resolved applicable list (by id+iteration identity, see + // holdoutSetMatches), used both to invalidate this cached assignment when coverage changes + // and to trigger each holdout's own exposure once this experiment is evaluated (see + // queueExposure). boolean suppressed; Experiment[] holdouts; Map variables = Collections.emptyMap(); @@ -764,30 +789,25 @@ private static class Assignment { final AtomicBoolean exposed = new AtomicBoolean(false); } - // Pins the holdout definition an Assignment was computed against, so getHoldoutAssignment can - // detect a refresh that changed the holdout's seed/split/iteration and recompute rather than - // reuse a stale membership verdict. Compares assignment-relevant fields only, mirroring - // experimentMatches for ordinary experiments. + // Pins the (iteration, unitType) a holdout's Assignment was computed against. This is the same + // granularity experimentMatches uses for ordinary experiments: seedHi/seedLo/split are + // deliberately excluded so a live seed or percentage edit - which does not change who is + // covered - never invalidates an already-exposed unit's arm. Only an iteration bump, a genuine + // re-randomization epoch, replaces the cached verdict (and its `exposed` flag), exactly as it + // does for ordinary experiments. private static class HoldoutAssignment { final Assignment assignment; final int iteration; - final int seedHi; - final int seedLo; - final double[] split; final String unitType; HoldoutAssignment(Assignment assignment, Experiment holdout, String unitType) { this.assignment = assignment; this.iteration = holdout.iteration; - this.seedHi = holdout.seedHi; - this.seedLo = holdout.seedLo; - this.split = holdout.split; this.unitType = unitType; } boolean matches(Experiment current, String currentUnitType) { - return (iteration == current.iteration) && (seedHi == current.seedHi) && (seedLo == current.seedLo) - && Arrays.equals(split, current.split) && unitType.equals(currentUnitType); + return (iteration == current.iteration) && unitType.equals(currentUnitType); } } @@ -971,9 +991,9 @@ private ContextExperiment getExperiment(final String experimentName) { // The holdout's own assignment is cached by holdout id rather than name, since holdout // entries live outside the experiments index and are shared by reference across every - // covered experiment. A definition change (seed, split, iteration) invalidates the cache - // entry so a refreshed holdout is re-assigned and re-exposed, exactly like a normal - // experiment's cached assignment does via experimentMatches. + // covered experiment. Only an iteration change invalidates the cache entry (see + // HoldoutAssignment), matching experimentMatches's treatment of ordinary experiments and + // guaranteeing an already-exposed unit's arm survives any seed, split or cosmetic edit. private Assignment getHoldoutAssignment(final Experiment holdout, final String unitType) { final String uid = units_.get(unitType); if (uid == null) { @@ -1278,6 +1298,18 @@ private void logError(Throwable error) { private final Map hashedUnits_; private final Map assigners_; private final Map assignmentCache_ = new HashMap(); + // A holdout's identity, for both this cache's keying and for detecting whether a covered + // experiment's own set of applicable holdouts is still current (holdoutSetMatches), is its + // (id, iteration) pair - never a full Experiment comparison. Rationale: the governing + // invariant is that a unit must never appear in both arms of the same holdout within one + // context's lifetime, which the once-per-context `exposed` AtomicBoolean on each cached + // Assignment enforces only as long as the entry it lives on is not needlessly replaced. + // seedHi/seedLo/split/name/variants/applications/audience/customFieldValues can all change + // without altering who is a member, so none of them may invalidate the entry - doing so + // would reset `exposed` and let an already-exposed unit be re-assigned into the other arm on + // the very next refresh. Only an iteration bump is a genuine re-randomization epoch and + // legitimately replaces the entry (and thus resets exposure), matching how experimentMatches + // already treats iteration for ordinary experiments. private final Map holdoutAssignmentCache_ = new HashMap(); private final ReentrantLock eventLock_ = new ReentrantLock(); diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 0eaecba..a6f2a7a 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -507,31 +507,133 @@ void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { assertEquals(1, context.getPendingCount()); // only the holdout's own exposure } + // A live seed edit within the same iteration is the crux of the bug this model fixes: naively + // comparing seedHi/seedLo (or the whole Experiment) invalidates the cached holdout Assignment, + // resets its `exposed` flag, and lets an already-exposed unit be re-assigned into the OTHER + // arm - variant 0 unit ends up exposed as variant 1 too, or vice versa. Same iteration must + // keep both the verdict and the exposure state pinned to the original arm. @Test - void refreshReassignsWhenHoldoutDefinitionChanges() { - final Experiment experiment = newExperiment(1, "exp_holdout_refresh"); + void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose() { + final Experiment experiment = newExperiment(1, "exp_holdout_seed_thrash"); - // unit is NOT held out initially (holdout B's seed) + // unit is held out initially (holdout A's seed, iteration 1) + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_seed_thrash")); // held out -> control + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure (variant 0) + + // same holdout id and iteration, but a seed edit that would flip this unit to variant 1 + // if it were re-assigned. + final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_seed_thrash"); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{reseededHoldout}, refreshedExperiment)); + refreshing.join(); + + // still variant 0 / held out: the pinned assignment from iteration 1 is reused, not + // recomputed against the new seed. + assertEquals(0, context.getTreatment("exp_holdout_seed_thrash")); + assertEquals(1, context.getPendingCount()); // no second, contradictory exposure in variant 1 + } + + // A cosmetic holdout edit - name, variants, applications - cannot change who is a member. + // Neither the holdout nor the covered experiment it suppresses may re-expose the unit: doing + // so would put the unit in variant 0 (already recorded) and, on the very same underlying + // assignment, effectively re-litigate variant 1 as well. + @Test + void refreshWithCosmeticHoldoutEditDoesNotReExposeHeldOutUnit() { + final Experiment experiment = newExperiment(1, "exp_holdout_cosmetic"); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic")); // held out + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure + + final Experiment cosmeticHoldout = newHoldout(11, "holdout_a_renamed", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + cosmeticHoldout.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; + cosmeticHoldout.variants = new ExperimentVariant[]{ + new ExperimentVariant("Control", null), new ExperimentVariant("HeldOut", null) + }; + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_cosmetic"); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{cosmeticHoldout}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic")); // still held out + assertEquals(1, context.getPendingCount()); // no new exposure for either the holdout or the experiment + } + + // The covered experiment side of the same guarantee: a non-held-out unit's own ordinary + // exposure must not be duplicated by a cosmetic holdout edit either. + @Test + void refreshWithCosmeticHoldoutEditDoesNotDuplicateCoveredExperimentExposure() { + final Experiment experiment = newExperiment(1, "exp_holdout_cosmetic_covered"); + final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic_covered")); // not held out + assertEquals(2, context.getPendingCount()); // own exposure + holdout exposure (variant 1) + + final Experiment cosmeticHoldout = newHoldout(11, "holdout_a_renamed", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_cosmetic_covered"); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{cosmeticHoldout}, refreshedExperiment)); + refreshing.join(); + + assertEquals(0, context.getTreatment("exp_holdout_cosmetic_covered")); + assertEquals(2, context.getPendingCount()); // no duplicate exposure of either kind + } + + // An iteration bump is the one holdout edit that legitimately changes membership: it is a new + // randomization epoch, exactly as it is for ordinary experiments (see + // refreshClearAssignmentCacheForIterationChange in ContextTest). The unit is re-assigned once + // against the new epoch and lands in exactly one arm of it; the prior epoch's exposure - a + // distinct, already-published fact about a now-superseded assignment - is not retroactively + // invalidated. Note that a seed/split change WITHOUT an iteration bump (same epoch) must NOT + // re-assign, which is exactly what distinguishes this test from the cosmetic-edit tests above. + @Test + void refreshWithIterationBumpReassignsToExactlyOneNewArm() { + final Experiment experiment = newExperiment(1, "exp_holdout_iteration"); + + // unit is NOT held out initially (holdout B's seed, iteration 1) final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, experiment)); - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_refresh")); + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_iteration")); assertEquals(2, context.getPendingCount()); // normal exposure + holdout exposure (variant 1) - // same holdout id, but the definition's seed changes so the unit is now held out - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_refresh"); + // same holdout id, new iteration with holdout A's seed: a genuine new epoch that now holds + // the unit out. + final Experiment newEpochHoldout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + newEpochHoldout.iteration = 2; + final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_iteration"); + final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); - final CompletableFuture refreshing = context.refreshAsync(); - refreshFuture.complete(contextDataOf( - new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, - refreshedExperiment)); + refreshFuture.complete(contextDataOf(new Experiment[]{newEpochHoldout}, refreshedExperiment)); refreshing.join(); - assertEquals(0, context.getTreatment("exp_holdout_refresh")); - // refreshed holdout re-exposes (its definition changed); the now-suppressed experiment - // emits no exposure of its own. + assertEquals(0, context.getTreatment("exp_holdout_iteration")); // new epoch: held out + // the new epoch's own exposure (variant 0); the now-suppressed experiment emits no + // exposure of its own for this epoch, so exactly one exposure is added. + assertEquals(3, context.getPendingCount()); + + // re-querying does not add a third exposure for the new epoch: exactly one arm was + // recorded for it, never both. + assertEquals(0, context.getTreatment("exp_holdout_iteration")); assertEquals(3, context.getPendingCount()); } From 959286b7cd95688551e9d599ebe07f737e6b124f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 13:28:05 +0000 Subject: [PATCH 15/49] fix: a suppressed assignment is not a participant in its experiment Setting `assigned = true` on the suppressed path made a held-out unit's assignment indistinguishable from a genuinely assigned one in two ways that only affect the held-out arm: 1. getVariableAssignment / getVariableValue reach queueExposure through `assigned || overridden`. A held-out unit satisfied that check via suppression and fired the holdout's own exposure on a variable lookup; a non-held-out unit whose strict-audience check failed left `assigned` false and never did, for the identical experiment and attributes - an emission asymmetry between the two arms of the same holdout. 2. Two experiments sharing a variable key are resolved by picking the first one with `assigned == true`. A suppressed experiment's forced `assigned = true` let it win that resolution over an experiment the unit is genuinely assigned to, hijacking the key. `assigned` now stays false for a suppressed assignment. Emission symmetry is restored by decoupling the holdout trigger from `assigned`: getVariableAssignment falls back to a suppressed match only when nothing else claims the key, so a held-out unit's variable lookup still reaches queueExposure (and so still fires the holdout's own exposure) without ever letting the suppressed experiment win over a real assignment. --- .../main/java/com/absmartly/sdk/Context.java | 20 ++++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 49 +++++++++++++++++++ 2 files changed, 66 insertions(+), 3 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index fe309df..821c927 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -753,8 +753,8 @@ private static boolean holdoutSetMatches(final Experiment[] a, final Experiment[ // variant, traffic ineligibility, or a strict audience mismatch — the custom value can never // equal that variant, so comparing the two would spuriously invalidate the cache and re-expose // on every getTreatment call. Treat those forced assignments as cache-valid regardless of the - // custom assignment. (A suppressed or forced assignment always has assigned=true except for the - // strict-mismatch and no-unit cases, where assigned stays false.) + // custom assignment. (A held-out unit is not a participant in the experiment, so `suppressed` + // is checked explicitly rather than relying on `assigned`.) private static boolean variantForcedRegardlessOfCustom(final Assignment assignment) { return assignment.suppressed || assignment.fullOn @@ -896,8 +896,10 @@ private Assignment getAssignment(final String experimentName) { assignment.suppressed = suppressed; if (suppressed) { + // A held-out unit is not a participant in this experiment: assigned stays + // false so it never wins variable-key resolution against a genuinely + // assigned experiment and never fires this experiment's own exposure. assignment.variant = 0; - assignment.assigned = true; } else { if (experiment.data.audience != null && experiment.data.audience.length() > 0) { final Map attrs = new HashMap(attributes_.size()); @@ -966,15 +968,27 @@ private Assignment getAssignment(final String experimentName) { } } + // A suppressed (held-out) experiment is not a participant, so it never wins resolution over + // a genuinely assigned or overridden experiment sharing the same variable key. It is used + // only as a fallback, so a held-out unit still reads control values and still triggers this + // experiment's holdouts — symmetric with the non-held-out path — when nothing else claims the + // key. private Assignment getVariableAssignment(final String key) { final List keyExperimentVariables = getVariableExperiments(key); if (keyExperimentVariables != null) { + Assignment suppressedFallback = null; for (final ContextExperiment experimentVariables : keyExperimentVariables) { final Assignment assignment = getAssignment(experimentVariables.data.name); if (assignment.assigned || assignment.overridden) { return assignment; } + if (assignment.suppressed && suppressedFallback == null) { + suppressedFallback = assignment; + } + } + if (suppressedFallback != null) { + return suppressedFallback; } } return null; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index a6f2a7a..3143163 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -742,4 +742,53 @@ void unitHeldOutByTwoHoldoutsSimultaneously() { holdoutExposure(VERDICT_UNIT_TYPE, 108, "holdout_108", 0)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + + // --- Regression tests for the coupled `assigned` / emission-symmetry fix ---------------- + // A suppressed assignment is not a participant in its experiment: `assigned` stays false. + // Reading a variable must still trigger the holdout's own exposure, for both arms, purely + // because the experiment was evaluated - regardless of whether that evaluation reached + // `getTreatment` or `getVariableValue`, and regardless of whether the experiment's own + // exposure was itself suppressed. + @Test + void variableLookupFiresHoldoutExposureForBothHeldOutAndNonHeldOutUnits() { + final Experiment experimentHeldOut = newExperiment(1, "exp_var_held_out"); + experimentHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; + final Context heldOutContext = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + experimentHeldOut)); + + // suppressed: control values only, but evaluating the variable must still trigger the + // holdout's own exposure. + assertEquals("default", heldOutContext.getVariableValue("var_a", "default")); + assertEquals(1, heldOutContext.getPendingCount()); // holdout exposure only + + final Experiment experimentNotHeldOut = newExperiment(1, "exp_var_not_held_out"); + experimentNotHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; + final Context notHeldOutContext = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + experimentNotHeldOut)); + + assertEquals("default", notHeldOutContext.getVariableValue("var_a", "default")); // normal variant is 0 + assertEquals(2, notHeldOutContext.getPendingCount()); // own exposure + holdout exposure + } + + // A held-out experiment must never win variable-key resolution over one the unit is + // genuinely assigned to, even when the held-out experiment has the lower id and is checked + // first. + @Test + void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { + final Experiment suppressedExperiment = newExperiment(1, "exp_var_key_suppressed"); + suppressedExperiment.variants[1].config = "{\"shared\":\"from_suppressed\"}"; + + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); + assignedExperiment.variants[1].config = "{\"shared\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", new int[]{2}); // excludes exp_var_key_assigned from coverage + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); + + assertEquals("from_assigned", context.getVariableValue("shared", "default")); + } } From 0395e7e76243d3fe38e57888fb5916cb7e31370d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 13:28:37 +0000 Subject: [PATCH 16/49] fix: read units_ under contextLock_ and avoid write lock on holdout cache hits getHoldoutAssignment read units_ before acquiring any lock, racing with setUnit's write-locked mutation of the same map. It also always took contextLock_'s exclusive write lock, serialising every holdout-covered evaluation in a context even when the result was already cached. The unit lookup and cache lookup now happen under the read lock, matching the pattern used elsewhere in this class (getUnit, setUnit). On a cache miss, the write lock is acquired and both lookups are repeated before computing a new assignment, so two threads racing on the same uncached holdout id can never install two different Assignment objects for it - the double-check preserves the once-per- context exposure invariant the (id, iteration) cache identity depends on. --- .../main/java/com/absmartly/sdk/Context.java | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 821c927..6be4613 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1009,15 +1009,34 @@ private ContextExperiment getExperiment(final String experimentName) { // HoldoutAssignment), matching experimentMatches's treatment of ordinary experiments and // guaranteeing an already-exposed unit's arm survives any seed, split or cosmetic edit. private Assignment getHoldoutAssignment(final Experiment holdout, final String unitType) { - final String uid = units_.get(unitType); - if (uid == null) { - return null; + final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); + try { + readLock.lock(); + + final String uid = units_.get(unitType); + if (uid == null) { + return null; + } + + final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); + if ((cached != null) && cached.matches(holdout, unitType)) { + return cached.assignment; + } + } finally { + readLock.unlock(); } + // Cache miss: recheck under the write lock before computing so two racing threads never + // install two different Assignment objects for the same holdout id. final ReentrantReadWriteLock.WriteLock writeLock = contextLock_.writeLock(); try { writeLock.lock(); + final String uid = units_.get(unitType); + if (uid == null) { + return null; + } + final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); if ((cached != null) && cached.matches(holdout, unitType)) { return cached.assignment; From 1eaeb4bd46bf14cee79218a180a25393367455f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 13:29:41 +0000 Subject: [PATCH 17/49] fix: normalize excludedExperimentIds and preserve holdout triggers on logger failure isExcluded binary-searches excludedExperimentIds, but the wire never guaranteed the array is sorted; an unsorted array made binary search return false negatives, silently letting a holdout wrongly cover an experiment it should have excluded (or vice versa). setData now sorts each holdout's excludedExperimentIds once at data-install time. queueExposure's single try-free call sequence meant a throwing ContextEventLogger aborted the whole method: `exposed` is CAS'd true before any of the per-holdout triggers run, so any holdout after the one that threw was skipped and could never be retried for the rest of the context's life - a permanent, silent loss of that holdout's exposure. The own-experiment enqueue and each holdout trigger are now individually guarded; every holdout still gets a chance to fire even if an earlier one's logger call throws, and the first failure is re-thrown once the loop completes so it is never swallowed. --- .../main/java/com/absmartly/sdk/Context.java | 32 +++++++++++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 39 +++++++++++++++++++ 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 6be4613..98c39be 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -382,19 +382,39 @@ public int getTreatment(@Nonnull final String experimentName) { // exposure - the holdout experiment's own exposure is the sole membership record. Firing // either exposure still triggers evaluation of every holdout applicable to this unit type, // keeping both holdout arms symmetric regardless of which covered experiment triggered it. + // The trigger loop below must run even if the exposure above throws (e.g. a + // ContextEventLogger implementation that throws): `exposed` is already CAS'd true by the + // time we get here, so a skipped holdout trigger would never be retried for this context's + // life. Failures are collected and re-thrown once every holdout has had a chance to fire, + // rather than swallowed or allowed to abort the loop early. private void queueExposure(final Assignment assignment) { if (assignment.exposed.compareAndSet(false, true)) { - if (!assignment.suppressed) { - enqueueExposure(assignment); + RuntimeException failure = null; + try { + if (!assignment.suppressed) { + enqueueExposure(assignment); + } + } catch (final RuntimeException e) { + failure = e; } if (assignment.holdouts != null) { for (final Experiment holdout : assignment.holdouts) { - queueHoldoutExposure(holdout, assignment.unitType); + try { + queueHoldoutExposure(holdout, assignment.unitType); + } catch (final RuntimeException e) { + if (failure == null) { + failure = e; + } + } } } setTimeout(); + + if (failure != null) { + throw failure; + } } } @@ -1199,6 +1219,12 @@ private void setData(final ContextData data) { if (data.holdouts != null) { for (final Experiment holdout : data.holdouts) { if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { + // isExcluded relies on binary search; the wire does not guarantee ordering, so + // normalize once here rather than on every lookup. + if (holdout.excludedExperimentIds != null) { + Arrays.sort(holdout.excludedExperimentIds); + } + List holdouts = holdoutsByUnitType.get(holdout.unitType); if (holdouts == null) { holdouts = new ArrayList(); diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 3143163..ba82902 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1,7 +1,9 @@ package com.absmartly.sdk; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; @@ -791,4 +793,41 @@ void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { assertEquals("from_assigned", context.getVariableValue("shared", "default")); } + + // Regression test for the unsorted-exclusion-array fix: the wire does not guarantee + // excludedExperimentIds is sorted, and isExcluded binary-searches it. setData must normalize + // the array so exclusions are applied correctly regardless of wire ordering. + @Test + void unsortedExcludedExperimentIdsStillExcludeCorrectly() { + final Experiment covered = newExperiment(1, "exp_holdout_in_unsorted"); + final Experiment excluded = newExperiment(2, "exp_excluded_unsorted"); + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", new int[]{9, 5, 2, 7}); // deliberately unsorted; excludes id 2 + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); + + assertEquals(0, context.getTreatment("exp_holdout_in_unsorted")); // suppressed -> control + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_excluded_unsorted")); // exclusion unaffected + } + + // Regression test for the logger-robustness fix: a throwing ContextEventLogger must not stop + // sibling holdout exposures from being queued. Holdout A (checked first, lower id) has a + // logger that throws; holdout B (checked second) must still fire. + @Test + void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), + org.mockito.ArgumentMatchers.argThat(o -> (o instanceof Exposure) && (((Exposure) o).id == 11))); + + final Experiment experiment = newExperiment(1, "exp_multi_holdout_throwing_logger"); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out, throws + newHoldout(12, "holdout_b", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out + }, experiment)); + + assertThrows(RuntimeException.class, () -> context.getTreatment("exp_multi_holdout_throwing_logger")); + + // both exposures were queued for publish despite holdout A's logger call throwing. + assertEquals(2, context.getPendingCount()); + } } From 28b31d153f2f19d09c98593690bdf73b51039609 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 14:30:17 +0000 Subject: [PATCH 18/49] test: prove the holdout union rule and harden malformed/missing-unit edge cases Add a multi-holdout test where the low-id holdout does not hold the unit out and a higher-id one does, verifying real assigner output rather than relying on ordering that let a first-holdout-only bug pass unnoticed. Add coverage for an experiment whose unit type has no configured unit at all, proving getHoldoutAssignment treats a missing unit as not evaluable instead of relying on an unexercised null guard. Add coverage for malformed holdout entries on the wire (a null array element, a null split, and an empty split), proving setData's filter drops each one rather than indexing it as applicable. --- .../com/absmartly/sdk/ContextHoldoutTest.java | 106 ++++++++++++++++++ 1 file changed, 106 insertions(+) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index ba82902..aff229c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -276,6 +276,30 @@ void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // The union rule is never proven if the deciding (held-out) holdout always sits at index 0 - + // a bug that stops at the first applicable holdout, or only ever consults holdouts[0], would + // still pass every other multi-holdout test above. Here the LOW-id holdout (11) does NOT hold + // the unit out and the HIGHER-id one (12) DOES; suppression must still trigger and both + // holdouts must still emit their own exposure with the correct variant. + @Test + void unionRuleAppliesWhenTheHigherIdHoldoutIsTheOnlyOneHoldingUnitOut() { + final Experiment experiment = newExperiment(1, "exp_union_high_id_decides"); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{ + newHoldout(11, "holdout_low_id", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out + newHoldout(12, "holdout_high_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out + }, experiment)); + + assertEquals(0, context.getTreatment("exp_union_high_id_decides")); // union -> suppressed + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_low_id", 1), + holdoutExposure(12, "holdout_high_id", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + @Test void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); @@ -376,6 +400,28 @@ void holdoutDoesNotApplyToDifferentUnitType() { assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_session")); } + // The unit type an experiment/holdout uses need not have any unit configured on this context + // at all (ContextConfig only sets session_id here). getHoldoutAssignment must treat a missing + // unit as "not evaluable" - control values, no exception, no holdout exposure - rather than + // NPE on the missing units_ entry. + @Test + void unconfiguredUnitTypeGetsControlValuesAndEmitsNoExposureAtAll() { + final Experiment experiment = newExperiment(1, "exp_user_holdout", "user_id", 0); + final Experiment holdout = newHoldout(11, "holdout_user", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", null); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID); // only session_id, no user_id + final Context context = createReadyContext(config, contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(0, context.peekTreatment("exp_user_holdout")); + + context.getTreatment("exp_user_holdout"); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + assertEquals(0, context.getPendingCount()); // neither the experiment nor the holdout fires + } + // (9): an absent holdouts key (null, the ContextData default) leaves behaviour identical to // pre-holdout: no suppression, no NPE. @Test @@ -830,4 +876,64 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { // both exposures were queued for publish despite holdout A's logger call throwing. assertEquals(2, context.getPendingCount()); } + + // setData silently drops malformed holdout entries (null, split==null, split empty) rather + // than indexing them: each case below places the malformed entry at the covered experiment's + // own unit type, so if the filter regressed to `holdout != null` alone, the malformed entry + // would become "applicable" and either NPE or crash inside VariantAssigner.assign. Instead + // the experiment must assign normally, as if no holdout existed at all. + @Test + void nullHoldoutArrayElementIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = newExperiment(1, "exp_null_holdout_entry"); + final Context context = createReadyContext(contextDataOf(new Experiment[]{null}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_null_holdout_entry")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_null_holdout_entry", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void holdoutWithNullSplitIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = newExperiment(1, "exp_null_split_holdout"); + final Experiment malformedHoldout = newHoldout(11, "holdout_null_split", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + malformedHoldout.split = null; + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{malformedHoldout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_null_split_holdout")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_null_split_holdout", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + @Test + void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = newExperiment(1, "exp_empty_split_holdout"); + final Experiment malformedHoldout = newHoldout(11, "holdout_empty_split", HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO); + malformedHoldout.split = new double[0]; + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{malformedHoldout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_empty_split_holdout")); + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_empty_split_holdout", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } } From b886623e41a12fd2356c1fdac5017dfc5bff1e60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sun, 9 Aug 2026 15:27:39 +0000 Subject: [PATCH 19/49] refactor: drop redundant suppressed check and rename exposure trigger methods variantForcedRegardlessOfCustom no longer checks assignment.suppressed: a suppressed assignment always has assigned == false, so !assignment.assigned already covers it. queueExposure/queueHoldoutExposure are renamed to triggerExposure/triggerHoldoutExposure to distinguish them from enqueueExposure, which is the one that actually appends to the pending exposure buffer. Also trims a few test comments that referenced prior review rounds or numbered an external spec instead of describing the invariant under test. --- .../main/java/com/absmartly/sdk/Context.java | 28 ++++++-------- .../com/absmartly/sdk/ContextHoldoutTest.java | 37 +++++++++---------- 2 files changed, 30 insertions(+), 35 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 98c39be..36f4d76 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -372,7 +372,7 @@ public int getTreatment(@Nonnull final String experimentName) { final Assignment assignment = getAssignment(experimentName); if (!assignment.exposed.get()) { - queueExposure(assignment); + triggerExposure(assignment); } return assignment.variant; @@ -387,7 +387,7 @@ public int getTreatment(@Nonnull final String experimentName) { // time we get here, so a skipped holdout trigger would never be retried for this context's // life. Failures are collected and re-thrown once every holdout has had a chance to fire, // rather than swallowed or allowed to abort the loop early. - private void queueExposure(final Assignment assignment) { + private void triggerExposure(final Assignment assignment) { if (assignment.exposed.compareAndSet(false, true)) { RuntimeException failure = null; try { @@ -401,7 +401,7 @@ private void queueExposure(final Assignment assignment) { if (assignment.holdouts != null) { for (final Experiment holdout : assignment.holdouts) { try { - queueHoldoutExposure(holdout, assignment.unitType); + triggerHoldoutExposure(holdout, assignment.unitType); } catch (final RuntimeException e) { if (failure == null) { failure = e; @@ -418,7 +418,7 @@ private void queueExposure(final Assignment assignment) { } } - private void queueHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { + private void triggerHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { final Assignment holdoutAssignment = getHoldoutAssignment(holdoutExperiment, unitType); if ((holdoutAssignment != null) && holdoutAssignment.exposed.compareAndSet(false, true)) { enqueueExposure(holdoutAssignment); @@ -486,7 +486,7 @@ public Object getVariableValue(@Nonnull final String key, final Object defaultVa if (assignment != null) { if (assignment.variables != null) { if (!assignment.exposed.get()) { - queueExposure(assignment); + triggerExposure(assignment); } if (assignment.variables.containsKey(key)) { @@ -769,17 +769,13 @@ private static boolean holdoutSetMatches(final Experiment[] a, final Experiment[ } // A custom assignment can only take effect on the normal, traffic-eligible assignment path. - // When the cached variant was forced by a higher-precedence rule — a holdout, a full-on - // variant, traffic ineligibility, or a strict audience mismatch — the custom value can never - // equal that variant, so comparing the two would spuriously invalidate the cache and re-expose - // on every getTreatment call. Treat those forced assignments as cache-valid regardless of the - // custom assignment. (A held-out unit is not a participant in the experiment, so `suppressed` - // is checked explicitly rather than relying on `assigned`.) + // When the cached variant was forced by a higher-precedence rule — a full-on variant, traffic + // ineligibility, a strict audience mismatch, or holdout suppression (which clears `assigned`) — + // the custom value can never equal that variant, so comparing the two would spuriously + // invalidate the cache and re-expose on every getTreatment call. Treat those forced + // assignments as cache-valid regardless of the custom assignment. private static boolean variantForcedRegardlessOfCustom(final Assignment assignment) { - return assignment.suppressed - || assignment.fullOn - || !assignment.eligible - || !assignment.assigned; + return assignment.fullOn || !assignment.eligible || !assignment.assigned; } private static class Assignment { @@ -801,7 +797,7 @@ private static class Assignment { // values only. `holdouts` is the resolved applicable list (by id+iteration identity, see // holdoutSetMatches), used both to invalidate this cached assignment when coverage changes // and to trigger each holdout's own exposure once this experiment is evaluated (see - // queueExposure). + // triggerExposure). boolean suppressed; Experiment[] holdouts; Map variables = Collections.emptyMap(); diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index aff229c..d113c73 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -177,8 +177,8 @@ Exposure holdoutExposure(String unitType, int id, String name, int variant) { return new Exposure(id, name, unitType, variant, clock.millis(), true, true, false, false, false, false); } - // (1) + (2): a held-out unit gets control values and emits zero exposures for the experiment - // it is held out of. + // A held-out unit gets control values and emits zero exposures for the experiment it is held + // out of. @Test void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { final Experiment experiment = newExperiment(1, "exp_holdout_in"); @@ -195,9 +195,8 @@ void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // (3): the holdout's own exposure is exactly one ordinary exposure with the unit's holdout - // variant and no holdout-specific fields, distinct from Exposure() itself no longer having - // heldOut/holdoutId fields at all. + // The holdout's own exposure is exactly one ordinary exposure with the unit's holdout + // variant and no holdout-specific fields. @Test void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { final Experiment experiment = newExperiment(1, "exp_holdout_in"); @@ -212,7 +211,7 @@ void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // (4): a non-held-out unit emits the holdout exposure with variant=1 plus its own normal + // A non-held-out unit emits the holdout exposure with variant=1 plus its own normal // exposure, unchanged. @Test void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { @@ -231,7 +230,7 @@ void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // (5): an experiment excluded from a holdout is never covered by it and emits normally for + // An experiment excluded from a holdout is never covered by it and emits normally for // both a held-out and a non-held-out unit, even while the same unit is suppressed elsewhere. @Test void excludedExperimentEmitsNormallyEvenForHeldOutUnit() { @@ -255,7 +254,7 @@ void excludedExperimentEmitsNormallyEvenForHeldOutUnit() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // (6): a normal experiment covered by two holdouts is suppressed if the unit is held out by + // A normal experiment covered by two holdouts is suppressed if the unit is held out by // EITHER one (union), and each applicable holdout still emits its own independent exposure. @Test void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { @@ -321,7 +320,7 @@ void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // (7): a full_on holdout is skipped entirely for a non-full-on experiment (no suppression, no + // A full_on holdout is skipped entirely for a non-full-on experiment (no suppression, no // exposure triggered), and applies normally to a full-on one; a `full` holdout applies to a // full-on experiment regardless of its fullOnVariant. @Test @@ -372,7 +371,7 @@ void fullHoldoutAppliesToFullOnExperimentRegardlessOfFullOnVariant() { assertEquals(0, context.getTreatment("exp_fullon_full_holdout")); // held out despite fullOnVariant=2 } - // (8): holdout applicability is derived from unit type alone; the `applications` field (which + // Holdout applicability is derived from unit type alone; the `applications` field (which // the wire contract leaves empty on holdout entries) plays no role in matching. @Test void applicabilityIgnoresApplicationsFieldOnBothSides() { @@ -422,7 +421,7 @@ void unconfiguredUnitTypeGetsControlValuesAndEmitsNoExposureAtAll() { assertEquals(0, context.getPendingCount()); // neither the experiment nor the holdout fires } - // (9): an absent holdouts key (null, the ContextData default) leaves behaviour identical to + // An absent holdouts key (null, the ContextData default) leaves behaviour identical to // pre-holdout: no suppression, no NPE. @Test void assignsNormallyWhenHoldoutsKeyIsAbsent() { @@ -443,7 +442,7 @@ void assignsNormallyWhenExperimentUnitTypeHasNoHoldouts() { assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_matching_holdouts")); } - // (10): a custom assignment can never override a held-out unit's variant - holdout precedence + // A custom assignment can never override a held-out unit's variant - holdout precedence // beats custom assignments, matching the existing audience/full-on/traffic precedence rules. @Test void customAssignmentCannotOverrideHeldOutVariant() { @@ -498,7 +497,7 @@ void holdoutTakesPrecedenceOverAudienceMismatch() { assertEquals(0, context.peekTreatment("exp_holdout_audience")); } - // (11): the holdout's own exposure is emitted once per context, not once per suppressed + // The holdout's own exposure is emitted once per context, not once per suppressed // experiment - two experiments covered by the same holdout still yield a single holdout // exposure. @Test @@ -555,11 +554,11 @@ void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { assertEquals(1, context.getPendingCount()); // only the holdout's own exposure } - // A live seed edit within the same iteration is the crux of the bug this model fixes: naively - // comparing seedHi/seedLo (or the whole Experiment) invalidates the cached holdout Assignment, - // resets its `exposed` flag, and lets an already-exposed unit be re-assigned into the OTHER - // arm - variant 0 unit ends up exposed as variant 1 too, or vice versa. Same iteration must - // keep both the verdict and the exposure state pinned to the original arm. + // A live seed edit within the same iteration must not change who is a member: naively + // comparing seedHi/seedLo (or the whole Experiment) would invalidate the cached holdout + // Assignment, reset its `exposed` flag, and let an already-exposed unit be re-assigned into + // the OTHER arm - variant 0 unit ends up exposed as variant 1 too, or vice versa. Same + // iteration must keep both the verdict and the exposure state pinned to the original arm. @Test void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose() { final Experiment experiment = newExperiment(1, "exp_holdout_seed_thrash"); @@ -791,7 +790,7 @@ void unitHeldOutByTwoHoldoutsSimultaneously() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // --- Regression tests for the coupled `assigned` / emission-symmetry fix ---------------- + // --- `assigned` and holdout-exposure symmetry --------------------------------------------- // A suppressed assignment is not a participant in its experiment: `assigned` stays false. // Reading a variable must still trigger the holdout's own exposure, for both arms, purely // because the experiment was evaluated - regardless of whether that evaluation reached From 66304fba3d8035594bdcd797acf003c74c1cd2da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 21 Aug 2026 15:05:03 +0000 Subject: [PATCH 20/49] fix: pin covered-experiment suppression to the same HoldoutAssignment as exposure Suppression was recomputing each applicable holdout's arm directly from the live seed/split/VariantAssigner, while getHoldoutAssignment pins that same holdout's arm by (id, iteration) for exposure purposes. A same-iteration seed refresh could flip the direct recomputation to a different arm than the one already pinned and exposed, letting an already-exposed holdout-variant-0 unit receive a covered experiment's treatment and exposure. Suppression now reads each holdout's arm from its pinned HoldoutAssignment, so exposure and suppression always agree. --- .../main/java/com/absmartly/sdk/Context.java | 24 +++++------ .../com/absmartly/sdk/ContextHoldoutTest.java | 43 +++++++++++++++++++ 2 files changed, 55 insertions(+), 12 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 36f4d76..4f536e7 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -894,18 +894,18 @@ private Assignment getAssignment(final String experimentName) { boolean suppressed = false; if (experiment.holdouts != null && experiment.holdouts.length > 0) { - final String uid = units_.get(unitType); - if (uid != null) { - final byte[] unitHash = Context.this.getUnitHash(unitType, uid); - final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, - unitHash); - // Union across every applicable holdout: variant 0 in any one of them - // suppresses this experiment's own exposure and forces control values. - for (final Experiment holdout : experiment.holdouts) { - if (assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo) == 0) { - suppressed = true; - break; - } + // Union across every applicable holdout: variant 0 in any one of them + // suppresses this experiment's own exposure and forces control values. Each + // holdout's arm is read from its pinned HoldoutAssignment (see + // getHoldoutAssignment) rather than recomputed from the live definition here, + // so suppression and the holdout's own exposure always agree, even after a + // same-iteration seed/split refresh. + for (final Experiment holdout : experiment.holdouts) { + final Assignment holdoutAssignment = Context.this.getHoldoutAssignment(holdout, + unitType); + if ((holdoutAssignment != null) && (holdoutAssignment.variant == 0)) { + suppressed = true; + break; } } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index d113c73..f1fb0ea 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -684,6 +684,49 @@ void refreshWithIterationBumpReassignsToExactlyOneNewArm() { assertEquals(3, context.getPendingCount()); } + // A same-iteration seed/split edit must not change who is suppressed, even for a covered + // experiment whose ordinary Assignment cache is a fresh miss (newly added, or invalidated). + // Suppression must come from the same pinned HoldoutAssignment as the holdout's own exposure, + // never recomputed directly from the live definition - otherwise a unit already exposed as + // holdout variant 0 could receive a covered experiment's treatment and exposure after the + // direct recomputation flips to variant 1. + @Test + void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedExperiment() { + final Experiment experimentA = newExperiment(1, "exp_holdout_desync_a"); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experimentA)); + + assertEquals(0, context.getTreatment("exp_holdout_desync_a")); // held out -> control + assertEquals(1, context.getPendingCount()); // holdout's own exposure (variant 0) + + // same holdout id and iteration, but a seed edit that would flip this unit to variant 1 + // if suppression were recomputed directly - plus a brand-new covered experiment whose + // Assignment cache has never been populated, forcing the write-lock computation path. + final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperimentA = newExperiment(1, "exp_holdout_desync_a"); + final Experiment experimentB = newExperiment(2, "exp_holdout_desync_b"); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{reseededHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // the pinned holdout arm (variant 0) must still govern suppression for the newly + // evaluated experiment, not the live seed's recomputed arm (which would be variant 1). + assertEquals(0, context.getTreatment("exp_holdout_desync_b")); + assertEquals(0, context.getTreatment("exp_holdout_desync_a")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // no new exposure at all: both experiments remain suppressed under the pinned arm, and + // the holdout's own exposure was already recorded before the refresh. + final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + // --- Cross-SDK parity vectors ----------------------------------------------------------- // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the From 2f88bfb381b3c1c2da796a767906f76637dd236d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 21 Aug 2026 15:06:11 +0000 Subject: [PATCH 21/49] fix: trigger applicable holdouts for every evaluated variable-key candidate getVariableAssignment resolves a variable key by walking every experiment that defines it, but only ever triggered exposure for the assignment ultimately returned. A lower-id covered experiment that was evaluated and suppressed - then lost the key to a later, genuinely assigned experiment - never fired its own applicable holdouts, violating the first-evaluation trigger contract. The non-peek path now fires each candidate's applicable holdouts as it is visited, while the ordinary experiment exposure is still emitted only for the selected assignment. peekVariableValue stays side- effect free by skipping the trigger entirely. --- .../main/java/com/absmartly/sdk/Context.java | 60 +++++++++++++++---- .../com/absmartly/sdk/ContextHoldoutTest.java | 32 ++++++++++ 2 files changed, 79 insertions(+), 13 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 4f536e7..0c1b0ac 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -398,16 +398,10 @@ private void triggerExposure(final Assignment assignment) { failure = e; } - if (assignment.holdouts != null) { - for (final Experiment holdout : assignment.holdouts) { - try { - triggerHoldoutExposure(holdout, assignment.unitType); - } catch (final RuntimeException e) { - if (failure == null) { - failure = e; - } - } - } + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment.holdouts, + assignment.unitType); + if (failure == null) { + failure = holdoutFailure; } setTimeout(); @@ -418,6 +412,27 @@ private void triggerExposure(final Assignment assignment) { } } + // Fires every holdout applicable to a unit type, independent of whether the covered + // experiment that surfaced them is the one ultimately selected for a treatment/variable + // lookup: the contract fires on first evaluation, not first selection. One throwing logger + // must not stop siblings, so failures are collected and the first one re-thrown only after + // every holdout has had a chance to fire. + private RuntimeException triggerApplicableHoldoutExposures(final Experiment[] holdouts, final String unitType) { + RuntimeException failure = null; + if (holdouts != null) { + for (final Experiment holdout : holdouts) { + try { + triggerHoldoutExposure(holdout, unitType); + } catch (final RuntimeException e) { + if (failure == null) { + failure = e; + } + } + } + } + return failure; + } + private void triggerHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { final Assignment holdoutAssignment = getHoldoutAssignment(holdoutExperiment, unitType); if ((holdoutAssignment != null) && holdoutAssignment.exposed.compareAndSet(false, true)) { @@ -482,7 +497,7 @@ public Map> getVariableKeys() { public Object getVariableValue(@Nonnull final String key, final Object defaultValue) { checkReady(true); - final Assignment assignment = getVariableAssignment(key); + final Assignment assignment = getVariableAssignment(key, false); if (assignment != null) { if (assignment.variables != null) { if (!assignment.exposed.get()) { @@ -500,7 +515,7 @@ public Object getVariableValue(@Nonnull final String key, final Object defaultVa public Object peekVariableValue(@Nonnull final String key, final Object defaultValue) { checkReady(true); - final Assignment assignment = getVariableAssignment(key); + final Assignment assignment = getVariableAssignment(key, true); if (assignment != null) { if (assignment.variables != null) { if (assignment.variables.containsKey(key)) { @@ -989,20 +1004,39 @@ private Assignment getAssignment(final String experimentName) { // only as a fallback, so a held-out unit still reads control values and still triggers this // experiment's holdouts — symmetric with the non-held-out path — when nothing else claims the // key. - private Assignment getVariableAssignment(final String key) { + // + // Every candidate reached while resolving the key was genuinely evaluated, whether or not it + // ends up the one returned, so on the non-peek path each candidate's applicable holdouts fire + // as it is visited - the ordinary experiment exposure is still emitted only for the winner, + // by the caller. peek must stay side-effect free, so it skips triggering entirely. + private Assignment getVariableAssignment(final String key, final boolean peek) { final List keyExperimentVariables = getVariableExperiments(key); if (keyExperimentVariables != null) { Assignment suppressedFallback = null; + RuntimeException failure = null; for (final ContextExperiment experimentVariables : keyExperimentVariables) { final Assignment assignment = getAssignment(experimentVariables.data.name); + if (!peek) { + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment.holdouts, + assignment.unitType); + if (failure == null) { + failure = holdoutFailure; + } + } if (assignment.assigned || assignment.overridden) { + if (failure != null) { + throw failure; + } return assignment; } if (assignment.suppressed && suppressedFallback == null) { suppressedFallback = assignment; } } + if (failure != null) { + throw failure; + } if (suppressedFallback != null) { return suppressedFallback; } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index f1fb0ea..e56090b 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -880,6 +880,38 @@ void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); assertEquals("from_assigned", context.getVariableValue("shared", "default")); + + // the lower-id suppressed experiment was evaluated to make this resolution decision, so + // its applicable holdout must fire even though its own value lost the key - the assigned + // experiment's own ordinary exposure fires too, but the suppressed one's does not. + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_a", 0), + new Exposure(2, "exp_var_key_assigned", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The peek path must stay side-effect free: resolving a variable key via peekVariableValue + // must not trigger any holdout exposure for a candidate it evaluates along the way. + @Test + void peekVariableValueNeverTriggersHoldoutExposureForEvaluatedCandidates() { + final Experiment suppressedExperiment = newExperiment(1, "exp_var_key_peek_suppressed"); + suppressedExperiment.variants[1].config = "{\"shared_peek\":\"from_suppressed\"}"; + + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); + assignedExperiment.variants[1].config = "{\"shared_peek\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", new int[]{2}); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); + + assertEquals("from_assigned", context.peekVariableValue("shared_peek", "default")); + assertEquals(0, context.getPendingCount()); // no exposure of any kind } // Regression test for the unsorted-exclusion-array fix: the wire does not guarantee From db9818f8b52e9c53ee4ecd62f7b310dc32df30b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 21 Aug 2026 15:06:48 +0000 Subject: [PATCH 22/49] fix: attach applicable holdouts to overridden assignments The override branch of getAssignment copied id and unitType but never attached experiment.holdouts, so getTreatment on an overridden experiment emitted its overridden exposure without evaluating any applicable holdout. An override only replaces the returned variant; the experiment is still evaluated, so its applicable holdouts must still fire, exactly as they do on the non-overridden path. --- .../main/java/com/absmartly/sdk/Context.java | 6 +++ .../com/absmartly/sdk/ContextHoldoutTest.java | 41 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 0c1b0ac..2ab6150 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -893,6 +893,12 @@ private Assignment getAssignment(final String experimentName) { if (experiment != null) { assignment.id = experiment.data.id; assignment.unitType = experiment.data.unitType; + + // An override still evaluates the experiment - only its variant is replaced - + // so the applicable holdouts must fire exactly as they would for a normal + // assignment. Override precedence itself is untouched: the returned variant + // stays the overridden one. + assignment.holdouts = experiment.holdouts; } assignment.overridden = true; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index e56090b..981375c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -484,6 +484,47 @@ void overrideTakesPrecedenceOverHoldout() { assertEquals(3, context.peekTreatment("exp_holdout_override")); } + // peekTreatment alone cannot prove the override branch attaches applicable holdouts, since + // peek never triggers exposures for anything. getTreatment on an overridden experiment must + // still trigger its applicable holdout's own exposure - for both arms - because the + // experiment was evaluated, exactly as it would be without the override. + @Test + void getTreatmentOnOverriddenExperimentTriggersApplicableHoldoutExposureBothArms() { + final Experiment heldOutExperiment = newExperiment(1, "exp_holdout_override_held_out"); + final ContextConfig heldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID) + .setOverride("exp_holdout_override_held_out", 3); + final Context heldOutContext = createReadyContext(heldOutConfig, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + heldOutExperiment)); + + assertEquals(3, heldOutContext.getTreatment("exp_holdout_override_held_out")); // override wins + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + heldOutContext.publish(); + + final PublishEvent expectedHeldOut = publishedEvent(UID, + new Exposure(1, "exp_holdout_override_held_out", UNIT_TYPE, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(11, "holdout_a", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(heldOutContext, expectedHeldOut); + + final Experiment notHeldOutExperiment = newExperiment(1, "exp_holdout_override_not_held_out"); + final ContextConfig notHeldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID_NOT_HELD_OUT) + .setOverride("exp_holdout_override_not_held_out", 3); + final Context notHeldOutContext = createReadyContext(notHeldOutConfig, contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, + notHeldOutExperiment)); + + assertEquals(3, notHeldOutContext.getTreatment("exp_holdout_override_not_held_out")); // override wins + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + notHeldOutContext.publish(); + + final PublishEvent expectedNotHeldOut = publishedEvent(UID_NOT_HELD_OUT, + new Exposure(1, "exp_holdout_override_not_held_out", UNIT_TYPE, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(11, "holdout_a", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(notHeldOutContext, expectedNotHeldOut); + } + @Test void holdoutTakesPrecedenceOverAudienceMismatch() { final Experiment experiment = newExperiment(1, "exp_holdout_audience"); From b63532e0936dfbdd7a6f737d07797868cf33bb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 21 Aug 2026 23:40:59 +0000 Subject: [PATCH 23/49] test: collapse identical variant ternary in holdout experiment factory Both branches constructed the same ExperimentVariant, which FindBugs reports as DB_DUPLICATE_BRANCHES and fails :core-api:findbugsTest under the JDK 8 toolchain used by CI. No caller requires a third variant. --- .../src/test/java/com/absmartly/sdk/ContextHoldoutTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 981375c..5a6f6f2 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -108,7 +108,7 @@ static Experiment newExperiment(int id, String name, String unitType, int fullOn experiment.applications = new ExperimentApplication[]{new ExperimentApplication("website")}; experiment.variants = new ExperimentVariant[]{ new ExperimentVariant("A", null), - fullOnVariant == 2 ? new ExperimentVariant("B", null) : new ExperimentVariant("B", null) + new ExperimentVariant("B", null) }; experiment.audienceStrict = false; experiment.audience = null; From b961b15e5e97ada68560a6269946d0b5b5c1b848 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 16:01:42 +0000 Subject: [PATCH 24/49] fix: resolve holdout definition from live data before pinned-cache checks getHoldoutAssignment trusted whichever Experiment object the caller passed when comparing against the pinned HoldoutAssignment and, on a cache miss, when computing a fresh one. A caller can hold a stale reference (e.g. a holdouts array captured before the most recent refresh) whose iteration no longer matches what is currently installed; recomputing from it would overwrite a cache entry a concurrent, genuinely newer evaluation already installed and exposed, producing a duplicate same-epoch exposure with a contradictory arm. Add an id-keyed index of the currently installed holdouts (holdoutsById_) and resolve the live definition by id before every matches()/compute step in getHoldoutAssignment, falling back to the caller-supplied reference only when the id is no longer present in the installed data. Id is stable across installs, so this removes any ambiguity about which definition is newer without weakening the (id, iteration) pinning itself. --- .../main/java/com/absmartly/sdk/Context.java | 52 ++++++++++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 72 +++++++++++++++++++ 2 files changed, 117 insertions(+), 7 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 2ab6150..0bd09b8 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1059,11 +1059,32 @@ private ContextExperiment getExperiment(final String experimentName) { } } + // Id-keyed lookup against the currently installed data, used to resolve a holdout's live + // definition regardless of which (possibly stale) Experiment reference a caller is holding. + private Experiment getHoldoutById(final int holdoutId) { + try { + dataLock_.readLock().lock(); + return holdoutsById_.get(holdoutId); + } finally { + dataLock_.readLock().unlock(); + } + } + // The holdout's own assignment is cached by holdout id rather than name, since holdout // entries live outside the experiments index and are shared by reference across every // covered experiment. Only an iteration change invalidates the cache entry (see // HoldoutAssignment), matching experimentMatches's treatment of ordinary experiments and // guaranteeing an already-exposed unit's arm survives any seed, split or cosmetic edit. + // + // The `holdout` parameter can be a stale Experiment reference: callers reach this method via + // a cached Assignment.holdouts array that may predate the most recent setData (e.g. the + // override fast path, or a refresh racing between getAssignment and triggerExposure). Trusting + // its iteration would let a dead definition overwrite a cache entry that a concurrent, + // genuinely newer evaluation already installed, producing a duplicate same-epoch exposure with + // a contradictory arm. Resolving by id against the currently-installed data before every + // matches()/compute step removes that ambiguity entirely: id is stable across installs, so the + // live lookup always reflects the newest definition, and a stale caller-supplied reference can + // never appear "newer" than what is actually installed. private Assignment getHoldoutAssignment(final Experiment holdout, final String unitType) { final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); try { @@ -1074,8 +1095,9 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u return null; } + final Experiment liveHoldout = resolveLiveHoldout(holdout); final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); - if ((cached != null) && cached.matches(holdout, unitType)) { + if ((cached != null) && cached.matches(liveHoldout, unitType)) { return cached.assignment; } } finally { @@ -1093,8 +1115,9 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u return null; } + final Experiment liveHoldout = resolveLiveHoldout(holdout); final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); - if ((cached != null) && cached.matches(holdout, unitType)) { + if ((cached != null) && cached.matches(liveHoldout, unitType)) { return cached.assignment; } @@ -1102,15 +1125,15 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u final VariantAssigner assigner = Context.this.getVariantAssigner(unitType, unitHash); final Assignment assignment = new Assignment(); - assignment.id = holdout.id; - assignment.name = holdout.name; - assignment.iteration = holdout.iteration; + assignment.id = liveHoldout.id; + assignment.name = liveHoldout.name; + assignment.iteration = liveHoldout.iteration; assignment.unitType = unitType; assignment.eligible = true; assignment.assigned = true; - assignment.variant = assigner.assign(holdout.split, holdout.seedHi, holdout.seedLo); + assignment.variant = assigner.assign(liveHoldout.split, liveHoldout.seedHi, liveHoldout.seedLo); - holdoutAssignmentCache_.put(holdout.id, new HoldoutAssignment(assignment, holdout, unitType)); + holdoutAssignmentCache_.put(liveHoldout.id, new HoldoutAssignment(assignment, liveHoldout, unitType)); return assignment; } finally { @@ -1118,6 +1141,14 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u } } + // Falls back to the caller-supplied reference only when the id is absent from the currently + // installed data (e.g. a holdout removed or invalidated by the latest refresh); there is no + // newer definition to prefer over it in that case. + private Experiment resolveLiveHoldout(final Experiment holdout) { + final Experiment live = Context.this.getHoldoutById(holdout.id); + return (live != null) ? live : holdout; + } + private List getVariableExperiments(final String key) { return Concurrency.getRW(dataLock_, indexVariables_, key); } @@ -1252,6 +1283,7 @@ private void setData(final ContextData data) { final Map> indexVariables = new HashMap>(); final Map> holdoutsByUnitType = new HashMap>(); + final Map holdoutsById = new HashMap(); if (data.holdouts != null) { for (final Experiment holdout : data.holdouts) { if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { @@ -1267,6 +1299,7 @@ private void setData(final ContextData data) { holdoutsByUnitType.put(holdout.unitType, holdouts); } holdouts.add(holdout); + holdoutsById.put(holdout.id, holdout); } } } @@ -1337,6 +1370,7 @@ public int compare(ContextExperiment a, ContextExperiment b) { index_ = index; indexVariables_ = indexVariables; + holdoutsById_ = holdoutsById; data_ = data; setRefreshTimer(); @@ -1350,6 +1384,7 @@ private void setDataFailed(final Throwable exception) { dataLock_.writeLock().lock(); index_ = new HashMap(); indexVariables_ = new HashMap>(); + holdoutsById_ = new HashMap(); data_ = new ContextData(); failed_ = true; } finally { @@ -1388,6 +1423,9 @@ private void logError(Throwable error) { private ContextData data_; private Map index_; private Map> indexVariables_; + // Id-keyed view of the holdouts installed by the most recent setData, used to resolve a + // holdout's live definition independent of any stale Experiment reference a caller holds. + private Map holdoutsById_; private final ReentrantReadWriteLock contextLock_ = new ReentrantReadWriteLock(); private final Map hashedUnits_; diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 5a6f6f2..1e10b64 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -8,6 +8,7 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.lang.reflect.Method; import java.util.concurrent.ScheduledExecutorService; import java8.util.concurrent.CompletableFuture; @@ -1051,4 +1052,75 @@ void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { false, false, false, false)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + + // --- Fable review fix regressions ----------------------------------------------------- + + // F1 (HIGH): a stale Experiment reference reaching getHoldoutAssignment must never overwrite + // a cache entry a concurrent, genuinely newer evaluation already installed and exposed. This + // reproduces the exact mechanism from the finding directly (bypassing the surrounding + // call-site plumbing via reflection into the private trigger path, since fixing finding #2 + // closes every call site that could reach this window through public API alone): the holdout + // H is bumped to a new iteration with a flipped verdict via the normal refresh + evaluation + // path, genuinely installing and exposing an it2 HoldoutAssignment (variant 1). A directly + // reconstructed, deliberately stale it1 Experiment object - same id, old iteration, the OLD + // seed that computes the OPPOSITE arm (variant 0) - is then fed straight into the private + // trigger path exactly as a lagging caller's cached holdouts array would. The fix must resolve + // H's live (it2) definition by id and return the already-exposed cache entry unchanged; a + // regression would recompute from the dead it1 seed, overwrite the it2 entry with a fresh, + // unexposed Assignment, and let it be exposed a second time with the opposite (contradictory) + // arm. + @Test + void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exception { + final Experiment holdoutIteration1 = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + final Experiment f = newExperiment(1, "exp_f"); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdoutIteration1}, f)); + + assertEquals(0, context.getTreatment("exp_f")); // held out by H@it1 (variant 0) + assertEquals(1, context.getPendingCount()); // H@it1's own exposure (variant 0) + + // Refresh: H bumped to a genuine new epoch (iteration 2) with a seed that flips the + // verdict to variant 1, covering a new experiment G. Evaluating G installs and exposes the + // it2 HoldoutAssignment for real, through the normal path. + final Experiment holdoutIteration2 = newHoldout(11, "holdout_h", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + holdoutIteration2.iteration = 2; + final Experiment refreshedF = newExperiment(1, "exp_f"); + final Experiment g = newExperiment(2, "exp_g"); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdoutIteration2}, refreshedF, g)); + refreshing.join(); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_g")); // not held out by H@it2 + assertEquals(3, context.getPendingCount()); // G's own exposure + H@it2's own exposure (variant 1) + + // A deliberately stale reference to H: same id, OLD iteration (1), and the OLD seed that + // computes the OPPOSITE arm from the live it2 definition. This is exactly what a lagging + // caller's cached holdouts array would still hold. + final Experiment staleHoldoutReference = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Method triggerHoldoutExposure = Context.class.getDeclaredMethod("triggerHoldoutExposure", + Experiment.class, String.class); + triggerHoldoutExposure.setAccessible(true); + triggerHoldoutExposure.invoke(context, staleHoldoutReference, UNIT_TYPE); + + // no second, contradictory exposure: the live it2 entry (already exposed, variant 1) must + // have been resolved and reused rather than overwritten by a fresh it1-seeded recompute. + assertEquals(3, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // f was suppressed under it1 and never emits an exposure of its own; H legitimately emits + // once per distinct epoch (it1 variant 0, it2 variant 1) - the invariant broken by the bug + // is a THIRD, contradictory exposure for the SAME it2 epoch, which is what is absent here. + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_h", 0), + new Exposure(2, "exp_g", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, false, + false), + holdoutExposure(11, "holdout_h", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } } From 5c090ec80c99f62bb01893afd71c62b7e4d7b1dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 16:02:17 +0000 Subject: [PATCH 25/49] fix: revalidate applicable-holdout coverage on the override fast path The cached-override fast path in getAssignment returned as soon as assignment.overridden && assignment.variant == override held, without ever checking holdoutSetMatches - unlike the ordinary branch, which invalidates via experimentMatches whenever coverage changes. A holdout becoming applicable to an overridden experiment after a refresh would therefore never fire for that experiment again. Add the same holdoutSetMatches check used by the ordinary path to the override fast path, so a coverage change invalidates an overridden assignment exactly as it would an ordinary one. --- .../main/java/com/absmartly/sdk/Context.java | 11 ++++- .../com/absmartly/sdk/ContextHoldoutTest.java | 43 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 0bd09b8..3ce533a 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -855,8 +855,15 @@ private Assignment getAssignment(final String experimentName) { final ContextExperiment experiment = Context.this.getExperiment(experimentName); if (override != null) { - if (assignment.overridden && assignment.variant == override) { - // override up-to-date + // An override still evaluates the experiment for holdout purposes (see the + // write path below), so the fast path must revalidate the applicable-holdout + // set exactly like the ordinary branch does via experimentMatches - otherwise + // a holdout that becomes applicable after a refresh never fires for an + // already-overridden experiment, and stays that way forever. + if (assignment.overridden && assignment.variant == override + && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, + assignment.holdouts)) { + // override and holdout coverage both up-to-date return assignment; } } else if (experiment == null) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 1e10b64..a9e5941 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1123,4 +1123,47 @@ void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exceptio holdoutExposure(11, "holdout_h", 1)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + + // F2 (MEDIUM): the override fast path must revalidate applicable-holdout coverage exactly + // like the ordinary experimentMatches branch does. A holdout that becomes applicable to an + // already-overridden, already-exposed experiment only after a refresh must still fire when + // that experiment is evaluated again - the override never suppresses holdout evaluation. + // Invalidating on the coverage change re-fires E's own exposure too, symmetric with how an + // ordinary experiment's experimentMatches-driven invalidation already behaves. + @Test + void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { + final Experiment experiment = newExperiment(1, "exp_override_late_holdout"); + + final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID) + .setOverride("exp_override_late_holdout", 3); + // no holdouts at all initially. + final Context context = createReadyContext(config, contextDataOf(experiment)); + + assertEquals(3, context.getTreatment("exp_override_late_holdout")); // override wins, no holdout yet + assertEquals(1, context.getPendingCount()); // only E's own exposure + + // refresh installs a holdout covering the same unit type/experiment for the first time. + final Experiment refreshedExperiment = newExperiment(1, "exp_override_late_holdout"); + final Experiment holdout = newHoldout(11, "holdout_late", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, refreshedExperiment)); + refreshing.join(); + + // re-evaluating the overridden experiment must still trigger the newly applicable + // holdout's own exposure. + assertEquals(3, context.getTreatment("exp_override_late_holdout")); // override still wins + assertEquals(3, context.getPendingCount()); // + E's re-fired exposure + the holdout's own + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final Exposure ownExposure = new Exposure(1, "exp_override_late_holdout", UNIT_TYPE, 3, clock.millis(), + false, true, true, false, false, false); + final PublishEvent expected = publishedEvent(UID, ownExposure, ownExposure, + holdoutExposure(11, "holdout_late", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } } From 52338898b21a6c9ee5b12f829678e8290f2fb170 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 16:03:25 +0000 Subject: [PATCH 26/49] fix: schedule a publish flush for every enqueued exposure setTimeout() was only called from triggerExposure, but enqueueExposure is also reached directly from the variable-key resolution path when a losing/suppressed candidate's applicable holdout fires while the winning assignment was already exposed. That path never called triggerExposure for the holdout's own exposure, so the enqueued exposure could sit unflushed until an unrelated event, an explicit publish(), or close(). Move the setTimeout() call into enqueueExposure itself, so every enqueued exposure - ordinary or holdout, from any call site - schedules its own flush. setTimeout() is already idempotent (guarded by timeout_ == null), so centralizing the call cannot cause double-scheduling. --- .../main/java/com/absmartly/sdk/Context.java | 10 ++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 32 +++++++++++++++++++ 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 3ce533a..b78ca1d 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -404,8 +404,6 @@ private void triggerExposure(final Assignment assignment) { failure = holdoutFailure; } - setTimeout(); - if (failure != null) { throw failure; } @@ -440,6 +438,12 @@ private void triggerHoldoutExposure(final Experiment holdoutExperiment, final St } } + // Every enqueued exposure - ordinary or holdout, from any of getTreatment, triggerExposure's + // holdout loop, or the variable-key path's per-candidate holdout firing - schedules its own + // flush here rather than relying on the caller: a caller can enqueue a holdout exposure + // without ever enqueueing its own (e.g. an already-exposed winner, or a losing variable-key + // candidate that only fires holdouts), and setTimeout() is idempotent, so centralizing the + // call is strictly safer than tracking every call site individually. private void enqueueExposure(final Assignment assignment) { final Exposure exposure = new Exposure(); exposure.id = assignment.id; @@ -463,6 +467,8 @@ private void enqueueExposure(final Assignment assignment) { } logEvent(ContextEventLogger.EventType.Exposure, exposure); + + setTimeout(); } public int peekTreatment(@Nonnull final String experimentName) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index a9e5941..1771e76 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -3,6 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -10,6 +11,7 @@ import java.lang.reflect.Method; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; import java8.util.concurrent.CompletableFuture; import org.junit.jupiter.api.BeforeEach; @@ -1166,4 +1168,34 @@ void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { holdoutExposure(11, "holdout_late", 0)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + + // F3 (MEDIUM): getVariableValue resolves via the variable-key path, which fires candidate + // holdout exposures as each candidate is visited. When the winning assignment was already + // exposed (so getVariableValue itself never calls triggerExposure -> setTimeout), a holdout + // exposure fired for a losing/suppressed candidate along the way must still schedule a flush + // on its own - it must not sit unflushed until an unrelated event. + @Test + void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { + final Experiment suppressed = newExperiment(1, "exp_var_flush_suppressed"); + suppressed.variants[1].config = "{\"flush_key\":\"from_suppressed\"}"; + + final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); + assigned.variants[1].config = "{\"flush_key\":\"from_assigned\"}"; + + final Experiment holdout = newHoldout(11, "holdout_flush", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, + "full", new int[]{2}); // excludes exp_var_flush_assigned from coverage + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, suppressed, assigned)); + + when(scheduler.schedule((Runnable) any(), eq(100L), eq(TimeUnit.MILLISECONDS))) + .thenReturn(mock(java.util.concurrent.ScheduledFuture.class)); + + assertEquals("from_assigned", context.getVariableValue("flush_key", "default")); + + // the suppressed candidate's applicable holdout fired an exposure that no other call path + // enqueued/exposed; a flush must have been scheduled for it regardless. + verify(scheduler, Mockito.timeout(5000).times(1)).schedule((Runnable) any(), eq(100L), + eq(TimeUnit.MILLISECONDS)); + } } From 070c77501c215247eacfff11a35ea62d2f1bd832 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 16:04:36 +0000 Subject: [PATCH 27/49] fix: sort a defensive copy of excludedExperimentIds instead of the wire object setData sorted holdout.excludedExperimentIds in place so isExcluded's binary search works, but the Experiment instance can be a caller-supplied, shared ContextData (ABSmartly.createContextWith). Sorting in place mutated the caller's own array, observable via getData()/equals and racy if the same ContextData instance were installed on two contexts concurrently. Add normalizeHoldout, which returns a shallow copy of the holdout carrying a private, sorted copy of excludedExperimentIds (the original array is reused unchanged when there is nothing to sort), and route setData's holdout indexing through it instead of sorting in place. --- .../main/java/com/absmartly/sdk/Context.java | 53 +++++++++++++++---- .../com/absmartly/sdk/ContextHoldoutTest.java | 22 ++++++++ 2 files changed, 66 insertions(+), 9 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index b78ca1d..5206915 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1253,6 +1253,41 @@ private static boolean isExcluded(final int[] excludedExperimentIds, final int e return (excludedExperimentIds != null) && (Arrays.binarySearch(excludedExperimentIds, experimentId) >= 0); } + // excludedExperimentIds must be sorted for isExcluded's binary search, but the wire does not + // guarantee ordering and the Experiment instance can be owned by the caller (ContextData + // supplied to ABSmartly.createContextWith, potentially shared/reused across contexts). + // Returns a shallow copy carrying a private, sorted copy of the array so isExcluded never + // mutates - or races on - the caller's data; the original is reused unchanged when there is + // nothing to sort. + private static Experiment normalizeHoldout(final Experiment holdout) { + if (holdout.excludedExperimentIds == null) { + return holdout; + } + + final Experiment copy = new Experiment(); + copy.id = holdout.id; + copy.name = holdout.name; + copy.unitType = holdout.unitType; + copy.iteration = holdout.iteration; + copy.seedHi = holdout.seedHi; + copy.seedLo = holdout.seedLo; + copy.split = holdout.split; + copy.trafficSeedHi = holdout.trafficSeedHi; + copy.trafficSeedLo = holdout.trafficSeedLo; + copy.trafficSplit = holdout.trafficSplit; + copy.fullOnVariant = holdout.fullOnVariant; + copy.applications = holdout.applications; + copy.variants = holdout.variants; + copy.audienceStrict = holdout.audienceStrict; + copy.audience = holdout.audience; + copy.customFieldValues = holdout.customFieldValues; + copy.holdoutType = holdout.holdoutType; + copy.excludedExperimentIds = Arrays.copyOf(holdout.excludedExperimentIds, holdout.excludedExperimentIds.length); + Arrays.sort(copy.excludedExperimentIds); + + return copy; + } + // A holdout applies to an experiment when their unit types match and the experiment is not // in the holdout's own exclusion list. A `full_on` holdout additionally applies only to // experiments that are themselves full-on (fullOnVariant != 0); `full` holdouts apply @@ -1300,19 +1335,19 @@ private void setData(final ContextData data) { if (data.holdouts != null) { for (final Experiment holdout : data.holdouts) { if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { - // isExcluded relies on binary search; the wire does not guarantee ordering, so - // normalize once here rather than on every lookup. - if (holdout.excludedExperimentIds != null) { - Arrays.sort(holdout.excludedExperimentIds); - } + // isExcluded relies on binary search, and the wire does not guarantee + // ordering. data.holdouts can be a caller-supplied, shared ContextData + // (ABSmartly.createContextWith), so normalize onto a private copy here rather + // than sorting the caller's array in place. + final Experiment normalized = normalizeHoldout(holdout); - List holdouts = holdoutsByUnitType.get(holdout.unitType); + List holdouts = holdoutsByUnitType.get(normalized.unitType); if (holdouts == null) { holdouts = new ArrayList(); - holdoutsByUnitType.put(holdout.unitType, holdouts); + holdoutsByUnitType.put(normalized.unitType, holdouts); } - holdouts.add(holdout); - holdoutsById.put(holdout.id, holdout); + holdouts.add(normalized); + holdoutsById.put(normalized.id, normalized); } } } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 1771e76..f656e05 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1,5 +1,6 @@ package com.absmartly.sdk; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; @@ -1198,4 +1199,25 @@ void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { verify(scheduler, Mockito.timeout(5000).times(1)).schedule((Runnable) any(), eq(100L), eq(TimeUnit.MILLISECONDS)); } + + // F4 (LOW): setData must not mutate a caller-supplied holdout's excludedExperimentIds array + // in place. The caller's array is asserted unchanged after setData runs, while exclusion + // behaviour (which relies on the sorted copy) still works correctly. + @Test + void setDataDoesNotMutateCallerSuppliedExcludedExperimentIdsArray() { + final Experiment covered = newExperiment(1, "exp_holdout_no_mutate_in"); + final Experiment excluded = newExperiment(2, "exp_holdout_no_mutate_excluded"); + final int[] callerArray = new int[]{9, 5, 2, 7}; + final Experiment holdout = newHoldout(11, "holdout_no_mutate", UNIT_TYPE, HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO, "full", callerArray); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); + + // exclusion still works correctly via the internally sorted copy. + assertEquals(0, context.getTreatment("exp_holdout_no_mutate_in")); // suppressed -> control + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_no_mutate_excluded")); // exclusion unaffected + + // the caller's own array, handed in via ContextData, must remain exactly as constructed. + assertArrayEquals(new int[]{9, 5, 2, 7}, callerArray); + } } From 83551cac73f3b2fc97f16a770b5523a7517dd775 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 19:08:33 +0000 Subject: [PATCH 28/49] feat: replace holdout-side excludedExperimentIds with per-experiment holdoutIds Experiment.holdoutIds now carries the ids of the holdouts that cover it, server-resolved. Context.resolveApplicableHoldouts looks each id up against the installed holdouts[] rather than deriving applicability from unit-type matching plus a full_on/exclusion rule; an id absent from holdouts[] simply contributes no coverage. isExcluded and normalizeHoldout are removed as dead code now that there is no exclusion list to binary-search or defensively sort. --- .../main/java/com/absmartly/sdk/Context.java | 87 ++++--------------- .../com/absmartly/sdk/json/Experiment.java | 12 ++- 2 files changed, 24 insertions(+), 75 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 5206915..d0dd2be 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1249,67 +1249,25 @@ private static class ContextCustomFieldValue { Object value; } - private static boolean isExcluded(final int[] excludedExperimentIds, final int experimentId) { - return (excludedExperimentIds != null) && (Arrays.binarySearch(excludedExperimentIds, experimentId) >= 0); - } - - // excludedExperimentIds must be sorted for isExcluded's binary search, but the wire does not - // guarantee ordering and the Experiment instance can be owned by the caller (ContextData - // supplied to ABSmartly.createContextWith, potentially shared/reused across contexts). - // Returns a shallow copy carrying a private, sorted copy of the array so isExcluded never - // mutates - or races on - the caller's data; the original is reused unchanged when there is - // nothing to sort. - private static Experiment normalizeHoldout(final Experiment holdout) { - if (holdout.excludedExperimentIds == null) { - return holdout; - } - - final Experiment copy = new Experiment(); - copy.id = holdout.id; - copy.name = holdout.name; - copy.unitType = holdout.unitType; - copy.iteration = holdout.iteration; - copy.seedHi = holdout.seedHi; - copy.seedLo = holdout.seedLo; - copy.split = holdout.split; - copy.trafficSeedHi = holdout.trafficSeedHi; - copy.trafficSeedLo = holdout.trafficSeedLo; - copy.trafficSplit = holdout.trafficSplit; - copy.fullOnVariant = holdout.fullOnVariant; - copy.applications = holdout.applications; - copy.variants = holdout.variants; - copy.audienceStrict = holdout.audienceStrict; - copy.audience = holdout.audience; - copy.customFieldValues = holdout.customFieldValues; - copy.holdoutType = holdout.holdoutType; - copy.excludedExperimentIds = Arrays.copyOf(holdout.excludedExperimentIds, holdout.excludedExperimentIds.length); - Arrays.sort(copy.excludedExperimentIds); - - return copy; - } - - // A holdout applies to an experiment when their unit types match and the experiment is not - // in the holdout's own exclusion list. A `full_on` holdout additionally applies only to - // experiments that are themselves full-on (fullOnVariant != 0); `full` holdouts apply - // regardless. This is derived once per experiment at data-install time, not per unit. + // An experiment's applicable holdouts are exactly the holdouts[] entries named by its + // holdoutIds, resolved once per experiment at data-install time, not per unit. A referenced + // id absent from holdoutsById (wire inconsistency, or a malformed holdout entry setData + // dropped during indexing) simply contributes no coverage rather than erroring - the id is + // treated as not present in holdouts[]. holdoutIds is only iterated here, never sorted or + // mutated, so no defensive copy of it is needed; the resulting applicable list is sorted by + // id for deterministic exposure ordering and to keep holdoutSetMatches comparisons stable. private static Experiment[] resolveApplicableHoldouts(final Experiment experiment, - final Map> holdoutsByUnitType) { - final List candidates = holdoutsByUnitType.get(experiment.unitType); - if (candidates == null || candidates.isEmpty()) { + final Map holdoutsById) { + if (experiment.holdoutIds == null || experiment.holdoutIds.length == 0) { return null; } - final List applicable = new ArrayList(candidates.size()); - for (final Experiment holdout : candidates) { - if ("full_on".equals(holdout.holdoutType) && experiment.fullOnVariant == 0) { - continue; - } - - if (isExcluded(holdout.excludedExperimentIds, experiment.id)) { - continue; + final List applicable = new ArrayList(experiment.holdoutIds.length); + for (final int holdoutId : experiment.holdoutIds) { + final Experiment holdout = holdoutsById.get(holdoutId); + if (holdout != null) { + applicable.add(holdout); } - - applicable.add(holdout); } if (applicable.isEmpty()) { @@ -1330,24 +1288,11 @@ private void setData(final ContextData data) { final Map index = new HashMap(); final Map> indexVariables = new HashMap>(); - final Map> holdoutsByUnitType = new HashMap>(); final Map holdoutsById = new HashMap(); if (data.holdouts != null) { for (final Experiment holdout : data.holdouts) { if ((holdout != null) && (holdout.split != null) && (holdout.split.length > 0)) { - // isExcluded relies on binary search, and the wire does not guarantee - // ordering. data.holdouts can be a caller-supplied, shared ContextData - // (ABSmartly.createContextWith), so normalize onto a private copy here rather - // than sorting the caller's array in place. - final Experiment normalized = normalizeHoldout(holdout); - - List holdouts = holdoutsByUnitType.get(normalized.unitType); - if (holdouts == null) { - holdouts = new ArrayList(); - holdoutsByUnitType.put(normalized.unitType, holdouts); - } - holdouts.add(normalized); - holdoutsById.put(normalized.id, normalized); + holdoutsById.put(holdout.id, holdout); } } } @@ -1355,7 +1300,7 @@ private void setData(final ContextData data) { for (final Experiment experiment : data.experiments) { final ContextExperiment contextExperiment = new ContextExperiment(); contextExperiment.data = experiment; - contextExperiment.holdouts = resolveApplicableHoldouts(experiment, holdoutsByUnitType); + contextExperiment.holdouts = resolveApplicableHoldouts(experiment, holdoutsById); contextExperiment.variables = new ArrayList>(experiment.variants.length); for (final ExperimentVariant variant : experiment.variants) { diff --git a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java index a8d2f24..f8173dd 100644 --- a/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java +++ b/core-api/src/main/java/com/absmartly/sdk/json/Experiment.java @@ -27,7 +27,11 @@ public class Experiment { // Set only for entries served in the top-level `holdouts` array; null for ordinary experiments. public String holdoutType; - public int[] excludedExperimentIds; + + // Set only for entries served in the top-level `experiments` array: the ids of the holdouts + // (from the top-level `holdouts` array) that cover this experiment. Null or empty means the + // experiment is not covered by any holdout. Absent on holdout entries themselves. + public int[] holdoutIds; public Experiment() {} @@ -74,7 +78,7 @@ public boolean equals(Object o) { return false; if (holdoutType != null ? !holdoutType.equals(that.holdoutType) : that.holdoutType != null) return false; - return Arrays.equals(excludedExperimentIds, that.excludedExperimentIds); + return Arrays.equals(holdoutIds, that.holdoutIds); } @Override @@ -96,7 +100,7 @@ public int hashCode() { result = 31 * result + (audience != null ? audience.hashCode() : 0); result = 31 * result + Arrays.hashCode(customFieldValues); result = 31 * result + (holdoutType != null ? holdoutType.hashCode() : 0); - result = 31 * result + Arrays.hashCode(excludedExperimentIds); + result = 31 * result + Arrays.hashCode(holdoutIds); return result; } @@ -120,7 +124,7 @@ public String toString() { ", audience='" + audience + '\'' + ", customFieldValues=" + Arrays.toString(customFieldValues) + ", holdoutType='" + holdoutType + '\'' + - ", excludedExperimentIds=" + Arrays.toString(excludedExperimentIds) + + ", holdoutIds=" + Arrays.toString(holdoutIds) + '}'; } } From 362b3d57c1c4a219b66e9aa7f1c13d930d6fb891 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 19:08:38 +0000 Subject: [PATCH 29/49] test: re-encode holdout fixtures as per-experiment holdoutIds Flips every ContextHoldoutTest fixture from setting excludedExperimentIds on the holdout entry to setting holdoutIds on the covered experiment via a new coveredBy helper; verdicts and suppression assertions are unchanged since coverage semantics only moved encoding. Adds coverage for a holdoutId absent from holdouts[] (ignored), an experiment with no holdoutIds (not covered), and reuses the existing two-holdout fixtures to prove simultaneous coverage still works under the new encoding. Updates the deserializer fixture and holdouts_context.json to match the wire shape. --- .../com/absmartly/sdk/ContextHoldoutTest.java | 277 +++++++----------- .../DefaultContextDataDeserializerTest.java | 3 +- .../src/test/resources/holdouts_context.json | 9 +- 3 files changed, 108 insertions(+), 181 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index f656e05..3006da7 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -32,8 +32,9 @@ // A holdout arrives as an ordinary experiment entry (in ContextData.holdouts, not .experiments) so // its variant semantics are: variant 0 = held out (no experimentation at all), variant 1 = exposed. -// Applicability to a covered experiment is derived client-side from unit type plus the full/full_on -// rule, minus that holdout's own excludedExperimentIds - there is no per-experiment holdoutIds field. +// A covered experiment declares its coverage explicitly via holdoutIds, an array of holdout ids +// resolved server-side; the SDK's only job is to look each id up in the installed holdouts[] and +// evaluate every one it finds (an id absent from holdouts[] is simply not covered by it). // A held-out unit gets control values and emits NO exposure for the experiments it covers; the // holdout experiment itself always emits one ordinary exposure, cached once per context. class ContextHoldoutTest extends TestUtils { @@ -120,11 +121,10 @@ static Experiment newExperiment(int id, String name, String unitType, int fullOn } static Experiment newHoldout(int id, String name, int seedHi, int seedLo) { - return newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "full", null); + return newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "full"); } - static Experiment newHoldout(int id, String name, String unitType, int seedHi, int seedLo, String holdoutType, - int[] excludedExperimentIds) { + static Experiment newHoldout(int id, String name, String unitType, int seedHi, int seedLo, String holdoutType) { final Experiment holdout = new Experiment(); holdout.id = id; holdout.name = name; @@ -143,10 +143,16 @@ static Experiment newHoldout(int id, String name, String unitType, int seedHi, i holdout.audienceStrict = false; holdout.audience = null; holdout.holdoutType = holdoutType; - holdout.excludedExperimentIds = excludedExperimentIds; return holdout; } + // Attaches coverage: the given holdout ids become this experiment's holdoutIds, exactly as a + // server-resolved wire payload would encode which holdouts apply to it. + static Experiment coveredBy(Experiment experiment, int... holdoutIds) { + experiment.holdoutIds = holdoutIds; + return experiment; + } + static ContextData contextDataOf(Experiment... experiments) { return contextDataOf(null, experiments); } @@ -185,7 +191,7 @@ Exposure holdoutExposure(String unitType, int id, String name, int variant) { // out of. @Test void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { - final Experiment experiment = newExperiment(1, "exp_holdout_in"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_in"), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -203,7 +209,7 @@ void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { // variant and no holdout-specific fields. @Test void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { - final Experiment experiment = newExperiment(1, "exp_holdout_in"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_in"), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -219,7 +225,7 @@ void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { // exposure, unchanged. @Test void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { - final Experiment experiment = newExperiment(1, "exp_holdout_out"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_out"), 11); final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -234,27 +240,27 @@ void notHeldOutUnitEmitsHoldoutExposureVariantOneAndNormalExposure() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // An experiment excluded from a holdout is never covered by it and emits normally for - // both a held-out and a non-held-out unit, even while the same unit is suppressed elsewhere. + // An experiment without holdoutIds is not covered by any holdout, even one that suppresses a + // sibling experiment for the same unit under the same context. @Test - void excludedExperimentEmitsNormallyEvenForHeldOutUnit() { - final Experiment covered = newExperiment(1, "exp_holdout_in"); - final Experiment excluded = newExperiment(2, "exp_excluded"); - final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full", - new int[]{2}); + void experimentWithoutHoldoutIdsEmitsNormallyEvenForHeldOutUnit() { + final Experiment covered = coveredBy(newExperiment(1, "exp_holdout_in"), 11); + final Experiment notCovered = newExperiment(2, "exp_not_covered"); // holdoutIds left null - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, notCovered)); assertEquals(0, context.getTreatment("exp_holdout_in")); // suppressed -> control - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_excluded")); // exclusion -> unaffected + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_not_covered")); // no coverage -> unaffected when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); context.publish(); final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0), - new Exposure(2, "exp_excluded", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, - false, false)); + new Exposure(2, "exp_not_covered", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } @@ -262,7 +268,7 @@ void excludedExperimentEmitsNormallyEvenForHeldOutUnit() { // EITHER one (union), and each applicable holdout still emits its own independent exposure. @Test void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { - final Experiment experiment = newExperiment(1, "exp_multi_holdout"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout"), 11, 12); final Context context = createReadyContext(contextDataOf( new Experiment[]{ newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out @@ -283,10 +289,12 @@ void unionOfApplicableHoldoutsSuppressesExperimentAndBothEmitOwnExposure() { // a bug that stops at the first applicable holdout, or only ever consults holdouts[0], would // still pass every other multi-holdout test above. Here the LOW-id holdout (11) does NOT hold // the unit out and the HIGHER-id one (12) DOES; suppression must still trigger and both - // holdouts must still emit their own exposure with the correct variant. + // holdouts must still emit their own exposure with the correct variant. This also pins the + // id-lookup itself: swapping which Experiment object id 11 vs id 12 resolves to would flip + // both the suppression verdict and which exposure carries which variant. @Test void unionRuleAppliesWhenTheHigherIdHoldoutIsTheOnlyOneHoldingUnitOut() { - final Experiment experiment = newExperiment(1, "exp_union_high_id_decides"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_high_id_decides"), 11, 12); final Context context = createReadyContext(contextDataOf( new Experiment[]{ newHoldout(11, "holdout_low_id", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO), // does not hold UID out @@ -305,7 +313,7 @@ void unionRuleAppliesWhenTheHigherIdHoldoutIsTheOnlyOneHoldingUnitOut() { @Test void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { - final Experiment experiment = newExperiment(1, "exp_multi_holdout_miss"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout_miss"), 11, 12); final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( new Experiment[]{ newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), @@ -324,83 +332,15 @@ void unitNotHeldOutByEitherHoldoutIsNotSuppressed() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // A full_on holdout is skipped entirely for a non-full-on experiment (no suppression, no - // exposure triggered), and applies normally to a full-on one; a `full` holdout applies to a - // full-on experiment regardless of its fullOnVariant. - @Test - void fullOnHoldoutSkippedForNonFullOnExperiment() { - final Experiment experiment = newExperiment(1, "exp_regular"); - final Context context = createReadyContext(contextDataOf( - new Experiment[]{newHoldout(11, "holdout_fullon", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full_on", null)}, - experiment)); - - // the unit would be held out (matches the holdout's split), but full_on holdouts only - // cover full-on experiments, so evaluation must proceed normally. - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_regular")); - - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); - - // the full_on holdout must not fire either - it never became applicable to any evaluated - // experiment in this context. - final PublishEvent expected = publishedEvent(UID, - new Exposure(1, "exp_regular", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, false, - false, false)); - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); - } - - @Test - void fullOnHoldoutAppliesToFullOnExperiment() { - final Experiment experiment = newExperiment(1, "exp_fullon", 2); - final Context context = createReadyContext(contextDataOf( - new Experiment[]{newHoldout(11, "holdout_fullon", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full_on", null)}, - experiment)); - - assertEquals(0, context.getTreatment("exp_fullon")); // suppressed -> control - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); - - final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_fullon", 0)); - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); - } - + // A holdout suppresses a full-on experiment exactly as it does a traffic-eligible one - + // coverage is declared explicitly via holdoutIds, so a full-on variant never bypasses it. @Test - void fullHoldoutAppliesToFullOnExperimentRegardlessOfFullOnVariant() { - final Experiment experiment = newExperiment(1, "exp_fullon_full_holdout", 2); + void holdoutSuppressesFullOnExperimentRegardlessOfFullOnVariant() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_fullon_holdout", 2), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - assertEquals(0, context.getTreatment("exp_fullon_full_holdout")); // held out despite fullOnVariant=2 - } - - // Holdout applicability is derived from unit type alone; the `applications` field (which - // the wire contract leaves empty on holdout entries) plays no role in matching. - @Test - void applicabilityIgnoresApplicationsFieldOnBothSides() { - final Experiment experiment = newExperiment(1, "exp_scoped"); - experiment.applications = new ExperimentApplication[]{new ExperimentApplication("mobile")}; - - final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - holdout.applications = null; // matches the wire contract's empty applications array - - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); - - assertEquals(0, context.peekTreatment("exp_scoped")); // still held out despite mismatched applications - } - - // A holdout only covers experiments sharing its unit type; a matching split/seed on a - // different unit type must never suppress. - @Test - void holdoutDoesNotApplyToDifferentUnitType() { - final Experiment experiment = newExperiment(1, "exp_session", UNIT_TYPE, 0); - final Experiment holdout = newHoldout(11, "holdout_user", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", null); - - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); - - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_session")); + assertEquals(0, context.getTreatment("exp_fullon_holdout")); // held out despite fullOnVariant=2 } // The unit type an experiment/holdout uses need not have any unit configured on this context @@ -409,9 +349,9 @@ void holdoutDoesNotApplyToDifferentUnitType() { // NPE on the missing units_ entry. @Test void unconfiguredUnitTypeGetsControlValuesAndEmitsNoExposureAtAll() { - final Experiment experiment = newExperiment(1, "exp_user_holdout", "user_id", 0); + final Experiment experiment = coveredBy(newExperiment(1, "exp_user_holdout", "user_id", 0), 11); final Experiment holdout = newHoldout(11, "holdout_user", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", null); + "full"); final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID); // only session_id, no user_id final Context context = createReadyContext(config, contextDataOf(new Experiment[]{holdout}, experiment)); @@ -435,22 +375,21 @@ void assignsNormallyWhenHoldoutsKeyIsAbsent() { assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_holdouts_key")); } + // A holdoutId that names no entry in holdouts[] is ignored - wire inconsistency tolerance, + // not an error - and the experiment assigns exactly as if it had no coverage at all. @Test - void assignsNormallyWhenExperimentUnitTypeHasNoHoldouts() { - final Experiment experiment = newExperiment(1, "exp_no_matching_holdouts"); - final Experiment holdout = newHoldout(11, "holdout_other", "user_id", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", null); - - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + void holdoutIdAbsentFromHoldoutsArrayIsIgnoredAndExperimentAssignsNormally() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_dangling_holdout_id"), 999); + final Context context = createReadyContext(contextDataOf(experiment)); // no holdouts[] entry with id 999 - assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_no_matching_holdouts")); + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_dangling_holdout_id")); } // A custom assignment can never override a held-out unit's variant - holdout precedence // beats custom assignments, matching the existing audience/full-on/traffic precedence rules. @Test void customAssignmentCannotOverrideHeldOutVariant() { - final Experiment experiment = newExperiment(1, "exp_holdout_custom"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_custom"), 11); final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( "exp_holdout_custom", 3); @@ -462,7 +401,7 @@ void customAssignmentCannotOverrideHeldOutVariant() { @Test void reusesCachedHeldOutAssignmentWithCustomAssignment() { - final Experiment experiment = newExperiment(1, "exp_holdout_custom_cache"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_custom_cache"), 11); final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setCustomAssignment( "exp_holdout_custom_cache", 3); @@ -478,7 +417,7 @@ void reusesCachedHeldOutAssignmentWithCustomAssignment() { @Test void overrideTakesPrecedenceOverHoldout() { - final Experiment experiment = newExperiment(1, "exp_holdout_override"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_override"), 11); final ContextConfig config = ContextConfig.create().setUnit(UNIT_TYPE, UID).setOverride( "exp_holdout_override", 3); @@ -494,7 +433,7 @@ void overrideTakesPrecedenceOverHoldout() { // experiment was evaluated, exactly as it would be without the override. @Test void getTreatmentOnOverriddenExperimentTriggersApplicableHoldoutExposureBothArms() { - final Experiment heldOutExperiment = newExperiment(1, "exp_holdout_override_held_out"); + final Experiment heldOutExperiment = coveredBy(newExperiment(1, "exp_holdout_override_held_out"), 11); final ContextConfig heldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID) .setOverride("exp_holdout_override_held_out", 3); final Context heldOutContext = createReadyContext(heldOutConfig, contextDataOf( @@ -511,7 +450,7 @@ void getTreatmentOnOverriddenExperimentTriggersApplicableHoldoutExposureBothArms holdoutExposure(11, "holdout_a", 0)); verify(eventHandler, Mockito.timeout(5000).times(1)).publish(heldOutContext, expectedHeldOut); - final Experiment notHeldOutExperiment = newExperiment(1, "exp_holdout_override_not_held_out"); + final Experiment notHeldOutExperiment = coveredBy(newExperiment(1, "exp_holdout_override_not_held_out"), 11); final ContextConfig notHeldOutConfig = ContextConfig.create().setUnit(UNIT_TYPE, UID_NOT_HELD_OUT) .setOverride("exp_holdout_override_not_held_out", 3); final Context notHeldOutContext = createReadyContext(notHeldOutConfig, contextDataOf( @@ -531,7 +470,7 @@ void getTreatmentOnOverriddenExperimentTriggersApplicableHoldoutExposureBothArms @Test void holdoutTakesPrecedenceOverAudienceMismatch() { - final Experiment experiment = newExperiment(1, "exp_holdout_audience"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_audience"), 11); experiment.audienceStrict = true; experiment.audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; @@ -547,8 +486,8 @@ void holdoutTakesPrecedenceOverAudienceMismatch() { // exposure. @Test void holdoutExposureEmittedOncePerContextNotPerSuppressedExperiment() { - final Experiment experimentA = newExperiment(1, "exp_shared_a"); - final Experiment experimentB = newExperiment(2, "exp_shared_b"); + final Experiment experimentA = coveredBy(newExperiment(1, "exp_shared_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_shared_b"), 11); final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experimentA, @@ -566,8 +505,8 @@ void holdoutExposureEmittedOncePerContextNotPerSuppressedExperiment() { @Test void holdoutExposureFiresOnceEvenWhenTriggeringExperimentIsNotSuppressed() { - final Experiment experimentA = newExperiment(1, "exp_shared_a"); - final Experiment experimentB = newExperiment(2, "exp_shared_b"); + final Experiment experimentA = coveredBy(newExperiment(1, "exp_shared_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_shared_b"), 11); final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final Context context = createReadyContext(UID_NOT_HELD_OUT, @@ -590,7 +529,7 @@ void holdoutExposureFiresOnceEvenWhenTriggeringExperimentIsNotSuppressed() { // change the covered experiment's cached suppression. @Test void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { - final Experiment experiment = newExperiment(1, "exp_holdout_cache"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cache"), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -606,7 +545,7 @@ void reusesCachedSuppressedAssignmentAcrossRepeatedCalls() { // iteration must keep both the verdict and the exposure state pinned to the original arm. @Test void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose() { - final Experiment experiment = newExperiment(1, "exp_holdout_seed_thrash"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_seed_thrash"), 11); // unit is held out initially (holdout A's seed, iteration 1) final Context context = createReadyContext(contextDataOf( @@ -618,7 +557,7 @@ void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose( // same holdout id and iteration, but a seed edit that would flip this unit to variant 1 // if it were re-assigned. final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_seed_thrash"); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_seed_thrash"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -638,7 +577,7 @@ void refreshWithSeedChangeSameIterationKeepsUnitInOriginalArmAndDoesNotReExpose( // assignment, effectively re-litigate variant 1 as well. @Test void refreshWithCosmeticHoldoutEditDoesNotReExposeHeldOutUnit() { - final Experiment experiment = newExperiment(1, "exp_holdout_cosmetic"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic"), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -651,7 +590,7 @@ void refreshWithCosmeticHoldoutEditDoesNotReExposeHeldOutUnit() { cosmeticHoldout.variants = new ExperimentVariant[]{ new ExperimentVariant("Control", null), new ExperimentVariant("HeldOut", null) }; - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_cosmetic"); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -667,7 +606,7 @@ void refreshWithCosmeticHoldoutEditDoesNotReExposeHeldOutUnit() { // exposure must not be duplicated by a cosmetic holdout edit either. @Test void refreshWithCosmeticHoldoutEditDoesNotDuplicateCoveredExperimentExposure() { - final Experiment experiment = newExperiment(1, "exp_holdout_cosmetic_covered"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic_covered"), 11); final Context context = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); @@ -676,7 +615,7 @@ void refreshWithCosmeticHoldoutEditDoesNotDuplicateCoveredExperimentExposure() { final Experiment cosmeticHoldout = newHoldout(11, "holdout_a_renamed", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_cosmetic_covered"); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_cosmetic_covered"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -697,7 +636,7 @@ void refreshWithCosmeticHoldoutEditDoesNotDuplicateCoveredExperimentExposure() { // re-assign, which is exactly what distinguishes this test from the cosmetic-edit tests above. @Test void refreshWithIterationBumpReassignsToExactlyOneNewArm() { - final Experiment experiment = newExperiment(1, "exp_holdout_iteration"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_iteration"), 11); // unit is NOT held out initially (holdout B's seed, iteration 1) final Context context = createReadyContext(contextDataOf( @@ -710,7 +649,7 @@ void refreshWithIterationBumpReassignsToExactlyOneNewArm() { // the unit out. final Experiment newEpochHoldout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); newEpochHoldout.iteration = 2; - final Experiment refreshedExperiment = newExperiment(1, "exp_holdout_iteration"); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_holdout_iteration"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -737,7 +676,7 @@ void refreshWithIterationBumpReassignsToExactlyOneNewArm() { // direct recomputation flips to variant 1. @Test void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedExperiment() { - final Experiment experimentA = newExperiment(1, "exp_holdout_desync_a"); + final Experiment experimentA = coveredBy(newExperiment(1, "exp_holdout_desync_a"), 11); final Context context = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experimentA)); @@ -748,8 +687,8 @@ void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedE // if suppression were recomputed directly - plus a brand-new covered experiment whose // Assignment cache has never been populated, forcing the write-lock computation path. final Experiment reseededHoldout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); - final Experiment refreshedExperimentA = newExperiment(1, "exp_holdout_desync_a"); - final Experiment experimentB = newExperiment(2, "exp_holdout_desync_b"); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_holdout_desync_a"), 11); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_holdout_desync_b"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -783,13 +722,18 @@ void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedE static final String VERDICT_UNIT_TYPE = "verdict_unit_type"; Context verdictContext(String uid, Experiment... holdouts) { - final Experiment experiment = newExperiment(9, "verdict_vectors_experiment", VERDICT_UNIT_TYPE, 0); + final int[] holdoutIds = new int[holdouts.length]; + for (int i = 0; i < holdouts.length; ++i) { + holdoutIds[i] = holdouts[i].id; + } + final Experiment experiment = coveredBy( + newExperiment(9, "verdict_vectors_experiment", VERDICT_UNIT_TYPE, 0), holdoutIds); final ContextConfig config = ContextConfig.create().setUnit(VERDICT_UNIT_TYPE, uid); return createReadyContext(config, contextDataOf(holdouts, experiment)); } static Experiment verdictHoldout(int id, int seedHi, int seedLo, double[] split) { - final Experiment holdout = newHoldout(id, "holdout_" + id, VERDICT_UNIT_TYPE, seedHi, seedLo, "full", null); + final Experiment holdout = newHoldout(id, "holdout_" + id, VERDICT_UNIT_TYPE, seedHi, seedLo, "full"); holdout.split = split; return holdout; } @@ -886,7 +830,7 @@ void unitHeldOutByTwoHoldoutsSimultaneously() { // exposure was itself suppressed. @Test void variableLookupFiresHoldoutExposureForBothHeldOutAndNonHeldOutUnits() { - final Experiment experimentHeldOut = newExperiment(1, "exp_var_held_out"); + final Experiment experimentHeldOut = coveredBy(newExperiment(1, "exp_var_held_out"), 11); experimentHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; final Context heldOutContext = createReadyContext(contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, @@ -897,7 +841,7 @@ void variableLookupFiresHoldoutExposureForBothHeldOutAndNonHeldOutUnits() { assertEquals("default", heldOutContext.getVariableValue("var_a", "default")); assertEquals(1, heldOutContext.getPendingCount()); // holdout exposure only - final Experiment experimentNotHeldOut = newExperiment(1, "exp_var_not_held_out"); + final Experiment experimentNotHeldOut = coveredBy(newExperiment(1, "exp_var_not_held_out"), 11); experimentNotHeldOut.variants[1].config = "{\"var_a\":\"value_a\"}"; final Context notHeldOutContext = createReadyContext(UID_NOT_HELD_OUT, contextDataOf( new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, @@ -912,14 +856,14 @@ void variableLookupFiresHoldoutExposureForBothHeldOutAndNonHeldOutUnits() { // first. @Test void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { - final Experiment suppressedExperiment = newExperiment(1, "exp_var_key_suppressed"); + final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_suppressed"), 11); suppressedExperiment.variants[1].config = "{\"shared\":\"from_suppressed\"}"; - final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); // not covered assignedExperiment.variants[1].config = "{\"shared\":\"from_assigned\"}"; final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", new int[]{2}); // excludes exp_var_key_assigned from coverage + "full"); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); @@ -943,14 +887,14 @@ void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { // must not trigger any holdout exposure for a candidate it evaluates along the way. @Test void peekVariableValueNeverTriggersHoldoutExposureForEvaluatedCandidates() { - final Experiment suppressedExperiment = newExperiment(1, "exp_var_key_peek_suppressed"); + final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_peek_suppressed"), 11); suppressedExperiment.variants[1].config = "{\"shared_peek\":\"from_suppressed\"}"; - final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); // not covered assignedExperiment.variants[1].config = "{\"shared_peek\":\"from_assigned\"}"; final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", new int[]{2}); + "full"); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); @@ -959,22 +903,6 @@ void peekVariableValueNeverTriggersHoldoutExposureForEvaluatedCandidates() { assertEquals(0, context.getPendingCount()); // no exposure of any kind } - // Regression test for the unsorted-exclusion-array fix: the wire does not guarantee - // excludedExperimentIds is sorted, and isExcluded binary-searches it. setData must normalize - // the array so exclusions are applied correctly regardless of wire ordering. - @Test - void unsortedExcludedExperimentIdsStillExcludeCorrectly() { - final Experiment covered = newExperiment(1, "exp_holdout_in_unsorted"); - final Experiment excluded = newExperiment(2, "exp_excluded_unsorted"); - final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", new int[]{9, 5, 2, 7}); // deliberately unsorted; excludes id 2 - - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); - - assertEquals(0, context.getTreatment("exp_holdout_in_unsorted")); // suppressed -> control - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_excluded_unsorted")); // exclusion unaffected - } - // Regression test for the logger-robustness fix: a throwing ContextEventLogger must not stop // sibling holdout exposures from being queued. Holdout A (checked first, lower id) has a // logger that throws; holdout B (checked second) must still fire. @@ -983,7 +911,7 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), org.mockito.ArgumentMatchers.argThat(o -> (o instanceof Exposure) && (((Exposure) o).id == 11))); - final Experiment experiment = newExperiment(1, "exp_multi_holdout_throwing_logger"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_multi_holdout_throwing_logger"), 11, 12); final Context context = createReadyContext(contextDataOf( new Experiment[]{ newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO), // holds UID out, throws @@ -1075,7 +1003,7 @@ void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { @Test void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exception { final Experiment holdoutIteration1 = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - final Experiment f = newExperiment(1, "exp_f"); + final Experiment f = coveredBy(newExperiment(1, "exp_f"), 11); final Context context = createReadyContext(contextDataOf(new Experiment[]{holdoutIteration1}, f)); @@ -1087,8 +1015,8 @@ void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exceptio // it2 HoldoutAssignment for real, through the normal path. final Experiment holdoutIteration2 = newHoldout(11, "holdout_h", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); holdoutIteration2.iteration = 2; - final Experiment refreshedF = newExperiment(1, "exp_f"); - final Experiment g = newExperiment(2, "exp_g"); + final Experiment refreshedF = coveredBy(newExperiment(1, "exp_f"), 11); + final Experiment g = coveredBy(newExperiment(2, "exp_g"), 11); final CompletableFuture refreshFuture = new CompletableFuture<>(); when(dataProvider.getContextData()).thenReturn(refreshFuture); @@ -1146,7 +1074,7 @@ void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { assertEquals(1, context.getPendingCount()); // only E's own exposure // refresh installs a holdout covering the same unit type/experiment for the first time. - final Experiment refreshedExperiment = newExperiment(1, "exp_override_late_holdout"); + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_override_late_holdout"), 11); final Experiment holdout = newHoldout(11, "holdout_late", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final CompletableFuture refreshFuture = new CompletableFuture<>(); @@ -1177,14 +1105,14 @@ void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { // on its own - it must not sit unflushed until an unrelated event. @Test void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { - final Experiment suppressed = newExperiment(1, "exp_var_flush_suppressed"); + final Experiment suppressed = coveredBy(newExperiment(1, "exp_var_flush_suppressed"), 11); suppressed.variants[1].config = "{\"flush_key\":\"from_suppressed\"}"; - final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); + final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); // not covered assigned.variants[1].config = "{\"flush_key\":\"from_assigned\"}"; final Experiment holdout = newHoldout(11, "holdout_flush", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full", new int[]{2}); // excludes exp_var_flush_assigned from coverage + "full"); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressed, assigned)); @@ -1200,24 +1128,25 @@ void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { eq(TimeUnit.MILLISECONDS)); } - // F4 (LOW): setData must not mutate a caller-supplied holdout's excludedExperimentIds array - // in place. The caller's array is asserted unchanged after setData runs, while exclusion - // behaviour (which relies on the sorted copy) still works correctly. + // F4 (LOW): setData must not mutate a caller-supplied experiment's holdoutIds array in + // place - resolution only ever reads it. The caller's array is asserted unchanged after + // setData runs, and coverage (which the resolution reads directly off it) still works. @Test - void setDataDoesNotMutateCallerSuppliedExcludedExperimentIdsArray() { + void setDataDoesNotMutateCallerSuppliedHoldoutIdsArray() { + final int[] callerArray = new int[]{12, 11}; // deliberately unsorted, and includes a dangling id (12) final Experiment covered = newExperiment(1, "exp_holdout_no_mutate_in"); - final Experiment excluded = newExperiment(2, "exp_holdout_no_mutate_excluded"); - final int[] callerArray = new int[]{9, 5, 2, 7}; - final Experiment holdout = newHoldout(11, "holdout_no_mutate", UNIT_TYPE, HOLDOUT_A_SEED_HI, - HOLDOUT_A_SEED_LO, "full", callerArray); + covered.holdoutIds = callerArray; + final Experiment notCovered = newExperiment(2, "exp_holdout_no_mutate_not_covered"); + final Experiment holdout = newHoldout(11, "holdout_no_mutate", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); - final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, excluded)); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, covered, notCovered)); - // exclusion still works correctly via the internally sorted copy. + // coverage still resolves correctly via id 11, and the dangling id 12 is simply ignored. assertEquals(0, context.getTreatment("exp_holdout_no_mutate_in")); // suppressed -> control - assertEquals(NORMAL_VARIANT, context.getTreatment("exp_holdout_no_mutate_excluded")); // exclusion unaffected + assertEquals(NORMAL_VARIANT, + context.getTreatment("exp_holdout_no_mutate_not_covered")); // no coverage -> unaffected // the caller's own array, handed in via ContextData, must remain exactly as constructed. - assertArrayEquals(new int[]{9, 5, 2, 7}, callerArray); + assertArrayEquals(new int[]{12, 11}, callerArray); } } diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index 263981c..ff0b41d 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -150,6 +150,7 @@ void deserializeHoldouts() { }; experiment.audienceStrict = false; experiment.audience = null; + experiment.holdoutIds = new int[]{11}; final Experiment holdoutA = new Experiment(); holdoutA.id = 11; @@ -168,7 +169,6 @@ void deserializeHoldouts() { holdoutA.audienceStrict = false; holdoutA.audience = null; holdoutA.holdoutType = "full"; - holdoutA.excludedExperimentIds = new int[0]; final Experiment holdoutB = new Experiment(); holdoutB.id = 12; @@ -187,7 +187,6 @@ void deserializeHoldouts() { holdoutB.audienceStrict = false; holdoutB.audience = null; holdoutB.holdoutType = "full_on"; - holdoutB.excludedExperimentIds = new int[]{4}; final ContextData expected = new ContextData( new Experiment[]{experiment}, diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json index b51da45..ccd14be 100644 --- a/core-api/src/test/resources/holdouts_context.json +++ b/core-api/src/test/resources/holdouts_context.json @@ -33,7 +33,8 @@ "config":"{\"banner.border\":1,\"banner.size\":\"large\"}" } ], - "audience": null + "audience": null, + "holdoutIds":[11] } ], "holdouts":[ @@ -66,8 +67,7 @@ } ], "audience": null, - "holdoutType":"full", - "excludedExperimentIds":[] + "holdoutType":"full" }, { "id":12, @@ -98,8 +98,7 @@ } ], "audience": null, - "holdoutType":"full_on", - "excludedExperimentIds":[4] + "holdoutType":"full_on" } ] } From 7fce871be07d4a9c795c52fef48687f4bfe58433 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 22:57:10 +0000 Subject: [PATCH 30/49] test: cover malformed-holdout filtering via explicit refs, drop review-history comments --- .../com/absmartly/sdk/ContextHoldoutTest.java | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 3006da7..2d5bc61 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -924,11 +924,16 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { assertEquals(2, context.getPendingCount()); } - // setData silently drops malformed holdout entries (null, split==null, split empty) rather - // than indexing them: each case below places the malformed entry at the covered experiment's - // own unit type, so if the filter regressed to `holdout != null` alone, the malformed entry - // would become "applicable" and either NPE or crash inside VariantAssigner.assign. Instead - // the experiment must assign normally, as if no holdout existed at all. + // setData silently drops malformed holdout entries (null, split==null, split empty) while + // building holdoutsById, rather than indexing them. The null-entry case below exercises that + // directly: a null element in data.holdouts must not NPE the indexing loop, regardless of + // whether any experiment references it. The split==null/split-empty cases instead cover + // resolveApplicableHoldouts' downstream behavior when a referenced id was dropped from + // holdoutsById at indexing time, so each covers its experiment by the malformed holdout's id + // (coveredBy(..., 11)) - if the split check regressed, the malformed holdout would stay + // indexed, resolveApplicableHoldouts would return it as applicable, and variant assignment + // would crash inside VariantAssigner.assign on the invalid split. Instead the experiment must + // assign normally, as if no holdout existed at all. @Test void nullHoldoutArrayElementIsIgnoredAndExperimentAssignsNormally() { final Experiment experiment = newExperiment(1, "exp_null_holdout_entry"); @@ -946,7 +951,7 @@ void nullHoldoutArrayElementIsIgnoredAndExperimentAssignsNormally() { @Test void holdoutWithNullSplitIsIgnoredAndExperimentAssignsNormally() { - final Experiment experiment = newExperiment(1, "exp_null_split_holdout"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_null_split_holdout"), 11); final Experiment malformedHoldout = newHoldout(11, "holdout_null_split", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); malformedHoldout.split = null; @@ -966,7 +971,7 @@ void holdoutWithNullSplitIsIgnoredAndExperimentAssignsNormally() { @Test void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { - final Experiment experiment = newExperiment(1, "exp_empty_split_holdout"); + final Experiment experiment = coveredBy(newExperiment(1, "exp_empty_split_holdout"), 11); final Experiment malformedHoldout = newHoldout(11, "holdout_empty_split", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); malformedHoldout.split = new double[0]; @@ -984,13 +989,13 @@ void holdoutWithEmptySplitIsIgnoredAndExperimentAssignsNormally() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // --- Fable review fix regressions ----------------------------------------------------- + // --- Holdout cache/publish invariants under concurrent refresh and overrides ------------ - // F1 (HIGH): a stale Experiment reference reaching getHoldoutAssignment must never overwrite - // a cache entry a concurrent, genuinely newer evaluation already installed and exposed. This - // reproduces the exact mechanism from the finding directly (bypassing the surrounding - // call-site plumbing via reflection into the private trigger path, since fixing finding #2 - // closes every call site that could reach this window through public API alone): the holdout + // A stale Experiment reference reaching getHoldoutAssignment must never overwrite a cache + // entry a concurrent, genuinely newer evaluation already installed and exposed. This + // reproduces the exact mechanism directly (bypassing the surrounding call-site plumbing via + // reflection into the private trigger path, since the public API closes every call site that + // could reach this window): the holdout // H is bumped to a new iteration with a flipped verdict via the normal refresh + evaluation // path, genuinely installing and exposing an it2 HoldoutAssignment (variant 1). A directly // reconstructed, deliberately stale it1 Experiment object - same id, old iteration, the OLD @@ -1055,8 +1060,8 @@ void staleHoldoutReferenceNeverOverwritesNewerPinnedCacheEntry() throws Exceptio verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // F2 (MEDIUM): the override fast path must revalidate applicable-holdout coverage exactly - // like the ordinary experimentMatches branch does. A holdout that becomes applicable to an + // The override fast path must revalidate applicable-holdout coverage exactly like the + // ordinary experimentMatches branch does. A holdout that becomes applicable to an // already-overridden, already-exposed experiment only after a refresh must still fire when // that experiment is evaluated again - the override never suppresses holdout evaluation. // Invalidating on the coverage change re-fires E's own exposure too, symmetric with how an @@ -1098,9 +1103,8 @@ void holdoutBecomingApplicableAfterRefreshStillFiresForOverriddenExperiment() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // F3 (MEDIUM): getVariableValue resolves via the variable-key path, which fires candidate - // holdout exposures as each candidate is visited. When the winning assignment was already - // exposed (so getVariableValue itself never calls triggerExposure -> setTimeout), a holdout + // getVariableValue resolves via the variable-key path, which fires candidate holdout + // exposures as each candidate is visited. When the winning assignment was already exposed (so getVariableValue itself never calls triggerExposure -> setTimeout), a holdout // exposure fired for a losing/suppressed candidate along the way must still schedule a flush // on its own - it must not sit unflushed until an unrelated event. @Test @@ -1128,9 +1132,9 @@ void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { eq(TimeUnit.MILLISECONDS)); } - // F4 (LOW): setData must not mutate a caller-supplied experiment's holdoutIds array in - // place - resolution only ever reads it. The caller's array is asserted unchanged after - // setData runs, and coverage (which the resolution reads directly off it) still works. + // setData must not mutate a caller-supplied experiment's holdoutIds array in place - + // resolution only ever reads it. The caller's array is asserted unchanged after setData + // runs, and coverage (which the resolution reads directly off it) still works. @Test void setDataDoesNotMutateCallerSuppliedHoldoutIdsArray() { final int[] callerArray = new int[]{12, 11}; // deliberately unsorted, and includes a dangling id (12) From 23072b73b9e4425f88793e9ae19e71594ceea373 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Sat, 22 Aug 2026 23:26:55 +0000 Subject: [PATCH 31/49] refactor: simplify holdout resolution/test machinery --- .../main/java/com/absmartly/sdk/Context.java | 9 +--- .../com/absmartly/sdk/ContextHoldoutTest.java | 48 ++++--------------- 2 files changed, 11 insertions(+), 46 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index d0dd2be..8c7aab3 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1249,13 +1249,8 @@ private static class ContextCustomFieldValue { Object value; } - // An experiment's applicable holdouts are exactly the holdouts[] entries named by its - // holdoutIds, resolved once per experiment at data-install time, not per unit. A referenced - // id absent from holdoutsById (wire inconsistency, or a malformed holdout entry setData - // dropped during indexing) simply contributes no coverage rather than erroring - the id is - // treated as not present in holdouts[]. holdoutIds is only iterated here, never sorted or - // mutated, so no defensive copy of it is needed; the resulting applicable list is sorted by - // id for deterministic exposure ordering and to keep holdoutSetMatches comparisons stable. + // Missing or malformed holdout references are ignored. Sort resolved holdouts by id for + // deterministic exposure ordering and stable cache matching. private static Experiment[] resolveApplicableHoldouts(final Experiment experiment, final Map holdoutsById) { if (experiment.holdoutIds == null || experiment.holdoutIds.length == 0) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 2d5bc61..1da53dd 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -146,8 +146,6 @@ static Experiment newHoldout(int id, String name, String unitType, int seedHi, i return holdout; } - // Attaches coverage: the given holdout ids become this experiment's holdoutIds, exactly as a - // server-resolved wire payload would encode which holdouts apply to it. static Experiment coveredBy(Experiment experiment, int... holdoutIds) { experiment.holdoutIds = holdoutIds; return experiment; @@ -187,8 +185,7 @@ Exposure holdoutExposure(String unitType, int id, String name, int variant) { return new Exposure(id, name, unitType, variant, clock.millis(), true, true, false, false, false, false); } - // A held-out unit gets control values and emits zero exposures for the experiment it is held - // out of. + // A held-out unit gets control values and emits only the holdout exposure. @Test void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_in"), 11); @@ -205,22 +202,6 @@ void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } - // The holdout's own exposure is exactly one ordinary exposure with the unit's holdout - // variant and no holdout-specific fields. - @Test - void heldOutUnitEmitsExactlyOneOrdinaryHoldoutExposure() { - final Experiment experiment = coveredBy(newExperiment(1, "exp_holdout_in"), 11); - final Context context = createReadyContext(contextDataOf( - new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); - - context.getTreatment("exp_holdout_in"); - when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); - context.publish(); - - final PublishEvent expected = publishedEvent(UID, holdoutExposure(11, "holdout_a", 0)); - verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); - } - // A non-held-out unit emits the holdout exposure with variant=1 plus its own normal // exposure, unchanged. @Test @@ -859,11 +840,10 @@ void variableKeyResolutionSkipsSuppressedAssignmentInFavorOfAssignedOne() { final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_suppressed"), 11); suppressedExperiment.variants[1].config = "{\"shared\":\"from_suppressed\"}"; - final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); // not covered + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_assigned"); assignedExperiment.variants[1].config = "{\"shared\":\"from_assigned\"}"; - final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full"); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); @@ -890,11 +870,10 @@ void peekVariableValueNeverTriggersHoldoutExposureForEvaluatedCandidates() { final Experiment suppressedExperiment = coveredBy(newExperiment(1, "exp_var_key_peek_suppressed"), 11); suppressedExperiment.variants[1].config = "{\"shared_peek\":\"from_suppressed\"}"; - final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); // not covered + final Experiment assignedExperiment = newExperiment(2, "exp_var_key_peek_assigned"); assignedExperiment.variants[1].config = "{\"shared_peek\":\"from_assigned\"}"; - final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full"); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressedExperiment, assignedExperiment)); @@ -924,16 +903,8 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { assertEquals(2, context.getPendingCount()); } - // setData silently drops malformed holdout entries (null, split==null, split empty) while - // building holdoutsById, rather than indexing them. The null-entry case below exercises that - // directly: a null element in data.holdouts must not NPE the indexing loop, regardless of - // whether any experiment references it. The split==null/split-empty cases instead cover - // resolveApplicableHoldouts' downstream behavior when a referenced id was dropped from - // holdoutsById at indexing time, so each covers its experiment by the malformed holdout's id - // (coveredBy(..., 11)) - if the split check regressed, the malformed holdout would stay - // indexed, resolveApplicableHoldouts would return it as applicable, and variant assignment - // would crash inside VariantAssigner.assign on the invalid split. Instead the experiment must - // assign normally, as if no holdout existed at all. + // Malformed holdouts are omitted from the id index; references to omitted entries must not + // affect normal experiment assignment. @Test void nullHoldoutArrayElementIsIgnoredAndExperimentAssignsNormally() { final Experiment experiment = newExperiment(1, "exp_null_holdout_entry"); @@ -1112,11 +1083,10 @@ void variablePathHoldoutExposureSchedulesFlushWithoutAnyOtherEvent() { final Experiment suppressed = coveredBy(newExperiment(1, "exp_var_flush_suppressed"), 11); suppressed.variants[1].config = "{\"flush_key\":\"from_suppressed\"}"; - final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); // not covered + final Experiment assigned = newExperiment(2, "exp_var_flush_assigned"); assigned.variants[1].config = "{\"flush_key\":\"from_assigned\"}"; - final Experiment holdout = newHoldout(11, "holdout_flush", UNIT_TYPE, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, - "full"); + final Experiment holdout = newHoldout(11, "holdout_flush", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); final Context context = createReadyContext( contextDataOf(new Experiment[]{holdout}, suppressed, assigned)); From 6a15eb9ca17eec6e01ed2131cf37b9680968d3c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 28 Aug 2026 00:21:23 +0000 Subject: [PATCH 32/49] feat: resolve holdout applicability from arm count instead of collector filtering --- .../main/java/com/absmartly/sdk/Context.java | 29 +++++++++++++++++-- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 8c7aab3..c29e1cc 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -928,8 +928,8 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, boolean suppressed = false; if (experiment.holdouts != null && experiment.holdouts.length > 0) { - // Union across every applicable holdout: variant 0 in any one of them - // suppresses this experiment's own exposure and forces control values. Each + // Union across every applicable holdout: any one of them holding this + // experiment out suppresses its own exposure and forces control values. Each // holdout's arm is read from its pinned HoldoutAssignment (see // getHoldoutAssignment) rather than recomputed from the live definition here, // so suppression and the holdout's own exposure always agree, even after a @@ -937,7 +937,8 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, for (final Experiment holdout : experiment.holdouts) { final Assignment holdoutAssignment = Context.this.getHoldoutAssignment(holdout, unitType); - if ((holdoutAssignment != null) && (holdoutAssignment.variant == 0)) { + if ((holdoutAssignment != null) && isHeldOutBy(holdout, holdoutAssignment.variant, + experiment.data.fullOnVariant)) { suppressed = true; break; } @@ -1018,6 +1019,28 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, } } + // Arm count comes from holdout.split.length, never from holdoutType. Per applicable holdout H + // and covered experiment X: H.variant==0 always holds X out. In a 3-arm H, variant==1 holds X + // out only when X.fullOnVariant==0 (X is not full-on); when X.fullOnVariant!=0, X defers to + // its normal assignment path exactly as variant==2 would, so audienceStrict is evaluated + // before the full-on variant is assigned and arm 1/2 never diverge for audience-unrelated + // reasons. A 2-arm H's variant==1 and a 3-arm H's variant==2 both defer to the normal path. A + // null or shorter-than-2 split cannot express arm 1's 3-arm meaning, so it is treated as + // not-held-out unless variant 0. + private static boolean isHeldOutBy(final Experiment holdout, final int holdoutVariant, + final int fullOnVariant) { + if (holdoutVariant == 0) { + return true; + } + + final int armCount = (holdout.split != null) ? holdout.split.length : 0; + if (armCount == 3 && holdoutVariant == 1) { + return fullOnVariant == 0; + } + + return false; + } + // A suppressed (held-out) experiment is not a participant, so it never wins resolution over // a genuinely assigned or overridden experiment sharing the same variable key. It is used // only as a fallback, so a held-out unit still reads control values and still triggers this From beb943fbf1bb84dcdd9aea4796eb95f545a28ba6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 28 Aug 2026 00:27:59 +0000 Subject: [PATCH 33/49] test: cover two- and three-arm holdout assignment --- .../com/absmartly/sdk/ContextHoldoutTest.java | 203 ++++++++++++++++++ .../DefaultContextDataDeserializerTest.java | 7 +- .../src/test/resources/holdouts_context.json | 9 +- 3 files changed, 214 insertions(+), 5 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 1da53dd..4e0a9cf 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -22,6 +22,7 @@ import com.absmartly.sdk.internal.hashing.Hashing; import com.absmartly.sdk.java.nio.charset.StandardCharsets; import com.absmartly.sdk.java.time.Clock; +import com.absmartly.sdk.json.Attribute; import com.absmartly.sdk.json.ContextData; import com.absmartly.sdk.json.Experiment; import com.absmartly.sdk.json.ExperimentApplication; @@ -56,6 +57,17 @@ class ContextHoldoutTest extends TestUtils { static final int HOLDOUT_B_SEED_HI = 1; static final int HOLDOUT_B_SEED_LO = 222; + // A 3-arm (all_full_on) holdout's split has length 3, which is what selects the arm-1 rule - + // holdoutType is never read by Context.java. Seeds below were found by exhaustive search + // against this split for UID: seedLo=1 -> arm 0, seedLo=3 -> arm 1, seedLo=0 -> arm 2. + static final double[] THREE_ARM_SPLIT = {0.3, 0.3, 0.4}; + static final int THREE_ARM_0_SEED_HI = 0; + static final int THREE_ARM_0_SEED_LO = 1; + static final int THREE_ARM_1_SEED_HI = 0; + static final int THREE_ARM_1_SEED_LO = 3; + static final int THREE_ARM_2_SEED_HI = 0; + static final int THREE_ARM_2_SEED_LO = 0; + ContextDataProvider dataProvider; ContextEventLogger eventLogger; ContextEventHandler eventHandler; @@ -146,6 +158,23 @@ static Experiment newHoldout(int id, String name, String unitType, int seedHi, i return holdout; } + // A 3-arm (all_full_on) holdout. holdoutType is set for wire-fidelity only; Context.java + // derives arity solely from split.length. + static Experiment newThreeArmHoldout(int id, String name, int seedHi, int seedLo) { + return newThreeArmHoldout(id, name, UNIT_TYPE, seedHi, seedLo); + } + + static Experiment newThreeArmHoldout(int id, String name, String unitType, int seedHi, int seedLo) { + final Experiment holdout = newHoldout(id, name, unitType, seedHi, seedLo, "all_full_on"); + holdout.split = THREE_ARM_SPLIT; + holdout.variants = new ExperimentVariant[]{ + new ExperimentVariant("A", null), + new ExperimentVariant("B", null), + new ExperimentVariant("C", null) + }; + return holdout; + } + static Experiment coveredBy(Experiment experiment, int... holdoutIds) { experiment.holdoutIds = holdoutIds; return experiment; @@ -324,6 +353,180 @@ void holdoutSuppressesFullOnExperimentRegardlessOfFullOnVariant() { assertEquals(0, context.getTreatment("exp_fullon_holdout")); // held out despite fullOnVariant=2 } + // --- Three-arm (all_full_on) holdouts ---------------------------------------------------- + // Arm 0 holds out every in-scope experiment, full-on or not, exactly like a two-arm holdout's + // variant 0. + @Test + void threeArmVariantZeroHoldsOutBothFullOnAndNonFullOnExperiments() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_0_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_0_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_0_SEED_HI, + THREE_ARM_0_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(0, context.getTreatment("exp_three_arm_0_non_fullon")); // held out -> control + assertEquals(0, context.getTreatment("exp_three_arm_0_fullon")); // held out despite fullOnVariant=2 + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, holdoutExposure(21, "holdout_three_arm", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 1 (full-on only) is the heart of the feature: a non-full-on experiment is forced to + // control, exactly as if held out, while a full-on experiment in the same holdout's scope is + // assigned its own fullOnVariant with fullOn=true and is NOT suppressed. + @Test + void threeArmVariantOneForcesNonFullOnExperimentToControlButAssignsFullOnExperimentNormally() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_1_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_1_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(0, context.getTreatment("exp_three_arm_1_non_fullon")); // forced to control + assertEquals(2, context.getTreatment("exp_three_arm_1_fullon")); // its own fullOnVariant, not suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // the non-full-on experiment emits no exposure of its own; the full-on one does, with + // fullOn=true; the holdout's own exposure fires once at variant 1. + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(21, "holdout_three_arm", 1), + new Exposure(2, "exp_three_arm_1_fullon", UNIT_TYPE, 2, clock.millis(), true, true, false, true, + false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 2 (normal traffic) evaluates every in-scope experiment completely normally, full-on or + // not, exactly as if uncovered. + @Test + void threeArmVariantTwoEvaluatesBothExperimentKindsNormally() { + final Experiment nonFullOn = coveredBy(newExperiment(1, "exp_three_arm_2_non_fullon"), 21); + final Experiment fullOn = coveredBy(newExperiment(2, "exp_three_arm_2_fullon", 2), 21); + final Experiment holdout = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdout}, nonFullOn, fullOn)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_three_arm_2_non_fullon")); // normal assignment + assertEquals(2, context.getTreatment("exp_three_arm_2_fullon")); // its own fullOnVariant + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_three_arm_2_non_fullon", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, + false, false, false, false), + holdoutExposure(21, "holdout_three_arm", 2), + new Exposure(2, "exp_three_arm_2_fullon", UNIT_TYPE, 2, clock.millis(), true, true, false, true, + false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Union: a 2-arm holdout that holds the unit out wins even when an applicable 3-arm holdout's + // arm (2, normal traffic) would not have held it out on its own. + @Test + void twoArmHoldoutWinsUnionEvenWhenThreeArmWouldNotHoldOut() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_two_arm_wins"), 11, 22); + final Experiment twoArmHoldout = newHoldout(11, "holdout_two_arm", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); // holds UID out + final Experiment threeArmHoldout = newThreeArmHoldout(22, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); // does not hold UID out + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{twoArmHoldout, threeArmHoldout}, experiment)); + + assertEquals(0, context.getTreatment("exp_union_two_arm_wins")); // union -> suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_two_arm", 0), + holdoutExposure(22, "holdout_three_arm", 2)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Union, the other direction: a 3-arm holdout's arm 1 holds a non-full-on experiment out even + // when an applicable 2-arm holdout would not have. + @Test + void threeArmHoldoutWinsUnionEvenWhenTwoArmWouldNotHoldOut() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_union_three_arm_wins"), 11, 22); + final Experiment twoArmHoldout = newHoldout(11, "holdout_two_arm", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // does not hold UID out + final Experiment threeArmHoldout = newThreeArmHoldout(22, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); // holds non-full-on UID out (arm 1) + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{twoArmHoldout, threeArmHoldout}, experiment)); + + assertEquals(0, context.getTreatment("exp_union_three_arm_wins")); // union -> suppressed + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + holdoutExposure(11, "holdout_two_arm", 1), + holdoutExposure(22, "holdout_three_arm", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Arm 1 must defer to the normal assignment path for a full-on experiment rather than + // short-circuiting to fullOnVariant: audienceStrict is still evaluated first, so an audience + // mismatch yields control (not the full-on variant) exactly as it would under arm 2 (normal + // traffic). The own exposure still fires - the experiment was evaluated and rejected by + // audience, not held out by the holdout - with audienceMismatch=true and assigned=false, + // identically for both arms. + @Test + void threeArmArmOneDefersToNormalPathSoAudienceMismatchStillWinsForFullOnExperiment() { + final String audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + + final Experiment fullOnArm1 = coveredBy(newExperiment(1, "exp_three_arm_1_audience_fullon", 2), 21); + fullOnArm1.audienceStrict = true; + fullOnArm1.audience = audience; + final Experiment holdoutArm1 = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + final Context contextArm1 = createReadyContext(contextDataOf(new Experiment[]{holdoutArm1}, fullOnArm1)); + contextArm1.setAttribute("age", 5); // mismatches the audience filter + + final Experiment fullOnArm2 = coveredBy(newExperiment(1, "exp_three_arm_2_audience_fullon", 2), 21); + fullOnArm2.audienceStrict = true; + fullOnArm2.audience = audience; + final Experiment holdoutArm2 = newThreeArmHoldout(21, "holdout_three_arm", THREE_ARM_2_SEED_HI, + THREE_ARM_2_SEED_LO); + final Context contextArm2 = createReadyContext(contextDataOf(new Experiment[]{holdoutArm2}, fullOnArm2)); + contextArm2.setAttribute("age", 5); // mismatches the audience filter + + // arm 1 (full-on only) and arm 2 (normal traffic) reach the identical verdict: audience + // mismatch forces control, not the full-on variant. + assertEquals(0, contextArm1.getTreatment("exp_three_arm_1_audience_fullon")); + assertEquals(0, contextArm2.getTreatment("exp_three_arm_2_audience_fullon")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + contextArm1.publish(); + contextArm2.publish(); + + final PublishEvent expectedArm1 = publishedEvent(UID, + new Exposure(1, "exp_three_arm_1_audience_fullon", UNIT_TYPE, 0, clock.millis(), false, true, false, + false, false, true), + holdoutExposure(21, "holdout_three_arm", 1)); + expectedArm1.attributes = new Attribute[]{new Attribute("age", 5, clock.millis())}; + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(contextArm1, expectedArm1); + + final PublishEvent expectedArm2 = publishedEvent(UID, + new Exposure(1, "exp_three_arm_2_audience_fullon", UNIT_TYPE, 0, clock.millis(), false, true, false, + false, false, true), + holdoutExposure(21, "holdout_three_arm", 2)); + expectedArm2.attributes = new Attribute[]{new Attribute("age", 5, clock.millis())}; + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(contextArm2, expectedArm2); + } + // The unit type an experiment/holdout uses need not have any unit configured on this context // at all (ContextConfig only sets session_id here). getHoldoutAssignment must treat a missing // unit as "not evaluable" - control values, no exception, no holdout exposure - rather than diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index ff0b41d..08812d5 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -177,16 +177,17 @@ void deserializeHoldouts() { holdoutB.iteration = 1; holdoutB.seedHi = 1; holdoutB.seedLo = 222; - holdoutB.split = new double[]{0.05, 0.95}; + holdoutB.split = new double[]{0.05, 0.05, 0.9}; holdoutB.trafficSplit = new double[]{0.0, 1.0}; holdoutB.fullOnVariant = 0; holdoutB.variants = new ExperimentVariant[]{ new ExperimentVariant("A", null), - new ExperimentVariant("B", null) + new ExperimentVariant("B", null), + new ExperimentVariant("C", null) }; holdoutB.audienceStrict = false; holdoutB.audience = null; - holdoutB.holdoutType = "full_on"; + holdoutB.holdoutType = "all_full_on"; final ContextData expected = new ContextData( new Experiment[]{experiment}, diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json index ccd14be..910778c 100644 --- a/core-api/src/test/resources/holdouts_context.json +++ b/core-api/src/test/resources/holdouts_context.json @@ -78,7 +78,8 @@ "seedLo":222, "split":[ 0.05, - 0.95 + 0.05, + 0.9 ], "trafficSeedHi":0, "trafficSeedLo":0, @@ -95,10 +96,14 @@ { "name":"B", "config":null + }, + { + "name":"C", + "config":null } ], "audience": null, - "holdoutType":"full_on" + "holdoutType":"all_full_on" } ] } From 14c905f30331ff2fb1fb11c21dc2b6808c7eeff3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 28 Aug 2026 02:16:01 +0000 Subject: [PATCH 34/49] fix: pin holdout arm count with the cached arm so refreshes cannot re-mean it --- .../main/java/com/absmartly/sdk/Context.java | 45 ++++--- .../com/absmartly/sdk/ContextHoldoutTest.java | 126 ++++++++++++++++++ 2 files changed, 153 insertions(+), 18 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index c29e1cc..5794605 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -807,6 +807,9 @@ private static class Assignment { String unitType; double[] trafficSplit; int variant; + // Arm count the holdout's own arm (variant, above) was computed against. Set only for a + // holdout's own Assignment (see getHoldoutAssignment); unused for ordinary experiments. + int armCount; boolean assigned; boolean overridden; boolean eligible; @@ -831,7 +834,10 @@ private static class Assignment { // deliberately excluded so a live seed or percentage edit - which does not change who is // covered - never invalidates an already-exposed unit's arm. Only an iteration bump, a genuine // re-randomization epoch, replaces the cached verdict (and its `exposed` flag), exactly as it - // does for ordinary experiments. + // does for ordinary experiments. armCount is deliberately excluded here too: it is pinned on + // Assignment itself (see armCount below), bundled with the arm number it was computed + // against, so the cached arm is always interpreted under its own arity rather than being + // invalidated and re-exposed under a new one for a same-iteration arity change. private static class HoldoutAssignment { final Assignment assignment; final int iteration; @@ -930,15 +936,15 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, if (experiment.holdouts != null && experiment.holdouts.length > 0) { // Union across every applicable holdout: any one of them holding this // experiment out suppresses its own exposure and forces control values. Each - // holdout's arm is read from its pinned HoldoutAssignment (see - // getHoldoutAssignment) rather than recomputed from the live definition here, - // so suppression and the holdout's own exposure always agree, even after a - // same-iteration seed/split refresh. + // holdout's arm AND the arm count it was computed against are read from its + // pinned HoldoutAssignment (see getHoldoutAssignment) rather than recomputed + // from the live definition here, so suppression and the holdout's own + // exposure always agree, even after a same-iteration seed/split refresh. for (final Experiment holdout : experiment.holdouts) { final Assignment holdoutAssignment = Context.this.getHoldoutAssignment(holdout, unitType); - if ((holdoutAssignment != null) && isHeldOutBy(holdout, holdoutAssignment.variant, - experiment.data.fullOnVariant)) { + if ((holdoutAssignment != null) && isHeldOutBy(holdoutAssignment.variant, + holdoutAssignment.armCount, experiment.data.fullOnVariant)) { suppressed = true; break; } @@ -1019,22 +1025,24 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, } } - // Arm count comes from holdout.split.length, never from holdoutType. Per applicable holdout H - // and covered experiment X: H.variant==0 always holds X out. In a 3-arm H, variant==1 holds X - // out only when X.fullOnVariant==0 (X is not full-on); when X.fullOnVariant!=0, X defers to - // its normal assignment path exactly as variant==2 would, so audienceStrict is evaluated - // before the full-on variant is assigned and arm 1/2 never diverge for audience-unrelated - // reasons. A 2-arm H's variant==1 and a 3-arm H's variant==2 both defer to the normal path. A - // null or shorter-than-2 split cannot express arm 1's 3-arm meaning, so it is treated as - // not-held-out unless variant 0. - private static boolean isHeldOutBy(final Experiment holdout, final int holdoutVariant, + // Arm count is the pinned count the holdout's own arm was computed against (Assignment.armCount + // from getHoldoutAssignment), never read live from holdout.split.length here: the live split + // can change within the same iteration (see HoldoutAssignment), and re-deriving arity from it + // would reinterpret an already-pinned arm number under a different meaning. Per applicable + // holdout H and covered experiment X: H.variant==0 always holds X out. In a 3-arm H, + // variant==1 holds X out only when X.fullOnVariant==0 (X is not full-on); when + // X.fullOnVariant!=0, X defers to its normal assignment path exactly as variant==2 would, so + // audienceStrict is evaluated before the full-on variant is assigned and arm 1/2 never diverge + // for audience-unrelated reasons. A 2-arm H's variant==1 and a 3-arm H's variant==2 both defer + // to the normal path. A null or shorter-than-2 split cannot express arm 1's 3-arm meaning, so + // it is treated as not-held-out unless variant 0. + private static boolean isHeldOutBy(final int holdoutVariant, final int holdoutArmCount, final int fullOnVariant) { if (holdoutVariant == 0) { return true; } - final int armCount = (holdout.split != null) ? holdout.split.length : 0; - if (armCount == 3 && holdoutVariant == 1) { + if (holdoutArmCount == 3 && holdoutVariant == 1) { return fullOnVariant == 0; } @@ -1168,6 +1176,7 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u assignment.eligible = true; assignment.assigned = true; assignment.variant = assigner.assign(liveHoldout.split, liveHoldout.seedHi, liveHoldout.seedLo); + assignment.armCount = (liveHoldout.split != null) ? liveHoldout.split.length : 0; holdoutAssignmentCache_.put(liveHoldout.id, new HoldoutAssignment(assignment, liveHoldout, unitType)); diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 4e0a9cf..0c85d7c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -895,6 +895,132 @@ void refreshedSeedDoesNotDesyncSuppressionFromPinnedHoldoutArmForNewlyEvaluatedE verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // --- Arm-count pinning across a same-iteration arity change ----------------------------- + // The arm number cached in HoldoutAssignment is meaningless without the arm count it was + // computed against: arm 1 means "defer to normal assignment" under a 2-arm holdout but + // "full-on only" under a 3-arm one. A same-iteration refresh that only changes split.length + // must not silently re-mean an already-pinned arm. + + // A unit pinned at arm 1 under a 2-arm holdout defers to normal assignment. If the holdout is + // refreshed to 3 arms within the same iteration, a newly evaluated non-full-on experiment must + // still defer to normal assignment (arm 1's pinned 2-arm meaning), not suddenly be suppressed + // as though arm 1 meant "full-on only" under the new, live arity. + @Test + void refreshFromTwoArmToThreeArmSameIterationDoesNotReinterpretDeferringArmAsFullOnOnly() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_arity_2to3_a"), 31); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(31, "holdout_arity", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, + experimentA)); + + // arm 1 under 2 arms: not held out, deferred to normal assignment. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_2to3_a")); + + // same holdout id and iteration, refreshed to 3 arms; a brand-new covered experiment is + // evaluated only after the refresh, forcing a fresh isHeldOutBy call against the pinned + // arm. + final Experiment threeArmHoldout = newThreeArmHoldout(31, "holdout_arity", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_arity_2to3_a"), 31); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_arity_2to3_b"), 31); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{threeArmHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // still deferred to normal assignment: the pinned arm is 1 under 2 arms, not under 3. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_2to3_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_arity_2to3_a", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(31, "holdout_arity", 1), + new Exposure(2, "exp_arity_2to3_b", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // The reverse direction: a unit pinned at arm 1 under a 3-arm holdout holds a non-full-on + // experiment out. If the holdout is refreshed to 2 arms within the same iteration, a newly + // evaluated non-full-on experiment must remain held out under arm 1's pinned 3-arm meaning, + // not be silently un-suppressed as though arm 1 meant "defer to normal" under the new, live + // arity. + @Test + void refreshFromThreeArmToTwoArmSameIterationKeepsFullOnOnlyArmSuppressingNonFullOnExperiment() { + final Experiment experimentA = coveredBy(newExperiment(1, "exp_arity_3to2_a"), 41); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newThreeArmHoldout(41, "holdout_arity_rev", THREE_ARM_1_SEED_HI, + THREE_ARM_1_SEED_LO)}, + experimentA)); + + // arm 1 under 3 arms: full-on only, so a non-full-on experiment is held out. + assertEquals(0, context.getTreatment("exp_arity_3to2_a")); + assertEquals(1, context.getPendingCount()); // only the holdout's own exposure (variant 1) + + // same holdout id and iteration, refreshed to 2 arms; a brand-new covered experiment is + // evaluated only after the refresh. + final Experiment twoArmHoldout = newHoldout(41, "holdout_arity_rev", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment refreshedExperimentA = coveredBy(newExperiment(1, "exp_arity_3to2_a"), 41); + final Experiment experimentB = coveredBy(newExperiment(2, "exp_arity_3to2_b"), 41); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{twoArmHoldout}, refreshedExperimentA, experimentB)); + refreshing.join(); + + // still held out: the pinned arm is 1 under 3 arms, not under 2. + assertEquals(0, context.getTreatment("exp_arity_3to2_b")); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + // neither covered experiment ever emits its own exposure; the holdout's own exposure + // fired once, before the refresh. + final PublishEvent expected = publishedEvent(UID, holdoutExposure(41, "holdout_arity_rev", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Within one context, an experiment evaluated before a same-iteration arity change and one + // evaluated only after it must not disagree about whether the pinned holdout arm holds them + // out: both are covered by the exact same HoldoutAssignment, so both must read the exact same + // pinned arm count. + @Test + void experimentsEvaluatedBeforeAndAfterSameIterationArityChangeAgreeOnSuppression() { + final Experiment experimentPre = coveredBy(newExperiment(1, "exp_arity_consistency_pre"), 71); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(71, "holdout_arity_consistency", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, + experimentPre)); + + // evaluated BEFORE the refresh: arm 1 under 2 arms, not held out. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_pre")); + + final Experiment threeArmHoldout = newThreeArmHoldout(71, "holdout_arity_consistency", + THREE_ARM_1_SEED_HI, THREE_ARM_1_SEED_LO); + final Experiment refreshedExperimentPre = coveredBy(newExperiment(1, "exp_arity_consistency_pre"), 71); + final Experiment experimentPost = coveredBy(newExperiment(2, "exp_arity_consistency_post"), 71); + + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete( + contextDataOf(new Experiment[]{threeArmHoldout}, refreshedExperimentPre, experimentPost)); + refreshing.join(); + + // evaluated ONLY AFTER the refresh, against the very same HoldoutAssignment: must agree + // with exp_arity_consistency_pre's own verdict rather than being re-meant under the new, + // live 3-arm split. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_post")); + // the pre-refresh verdict is unchanged by the refresh. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_pre")); + } + // --- Cross-SDK parity vectors ----------------------------------------------------------- // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the From 07c0ae716506c8a72f6b0ba503152e148807d768 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Fri, 28 Aug 2026 04:07:52 +0000 Subject: [PATCH 35/49] refactor: simplify holdout arm resolution --- .../src/test/java/com/absmartly/sdk/ContextHoldoutTest.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 0c85d7c..4cddd8b 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -161,11 +161,7 @@ static Experiment newHoldout(int id, String name, String unitType, int seedHi, i // A 3-arm (all_full_on) holdout. holdoutType is set for wire-fidelity only; Context.java // derives arity solely from split.length. static Experiment newThreeArmHoldout(int id, String name, int seedHi, int seedLo) { - return newThreeArmHoldout(id, name, UNIT_TYPE, seedHi, seedLo); - } - - static Experiment newThreeArmHoldout(int id, String name, String unitType, int seedHi, int seedLo) { - final Experiment holdout = newHoldout(id, name, unitType, seedHi, seedLo, "all_full_on"); + final Experiment holdout = newHoldout(id, name, UNIT_TYPE, seedHi, seedLo, "all_full_on"); holdout.split = THREE_ARM_SPLIT; holdout.variants = new ExperimentVariant[]{ new ExperimentVariant("A", null), From 55b87fe994a24222d039f79d84dc73456cb9b2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 13:45:19 +0000 Subject: [PATCH 36/49] fix: trigger holdout exposures from the assignments used for the decision getAssignment now resolves every applicable holdout up front and pins the resulting HoldoutAssignment objects on the Assignment (holdoutAssignments), instead of stopping at the first suppressing holdout. triggerExposure fires that pinned snapshot rather than re-resolving the holdout definitions at exposure time, so a refresh landing between the suppression decision and the exposure trigger can no longer publish an exposure pair from two different holdout iterations. --- .../main/java/com/absmartly/sdk/Context.java | 65 +++++++++++++------ .../com/absmartly/sdk/ContextHoldoutTest.java | 49 ++++++++++++++ 2 files changed, 95 insertions(+), 19 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 5794605..e0f19a2 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -398,8 +398,7 @@ private void triggerExposure(final Assignment assignment) { failure = e; } - final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment.holdouts, - assignment.unitType); + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment); if (failure == null) { failure = holdoutFailure; } @@ -410,17 +409,28 @@ private void triggerExposure(final Assignment assignment) { } } - // Fires every holdout applicable to a unit type, independent of whether the covered + // Fires every holdout applicable to the given assignment, independent of whether the covered // experiment that surfaced them is the one ultimately selected for a treatment/variable - // lookup: the contract fires on first evaluation, not first selection. One throwing logger - // must not stop siblings, so failures are collected and the first one re-thrown only after - // every holdout has had a chance to fire. - private RuntimeException triggerApplicableHoldoutExposures(final Experiment[] holdouts, final String unitType) { + // lookup: the contract fires on first evaluation, not first selection. When the assignment + // carries a pinned holdoutAssignments snapshot (see Assignment.holdoutAssignments), each + // entry is fired directly rather than re-resolved, so the exposure always reflects the exact + // epoch the suppression decision was made from - a refresh landing between decision and + // trigger can never publish a holdout exposure from a different iteration than the one that + // governed this assignment. The override path never takes that snapshot, so it falls back to + // a live-by-id resolution. One throwing logger must not stop siblings, so failures are + // collected and the first one re-thrown only after every holdout has had a chance to fire. + private RuntimeException triggerApplicableHoldoutExposures(final Assignment assignment) { RuntimeException failure = null; + final Experiment[] holdouts = assignment.holdouts; if (holdouts != null) { - for (final Experiment holdout : holdouts) { + final Assignment[] pinned = assignment.holdoutAssignments; + for (int i = 0; i < holdouts.length; ++i) { try { - triggerHoldoutExposure(holdout, unitType); + if (pinned != null) { + triggerHoldoutExposure(pinned[i]); + } else { + triggerHoldoutExposure(holdouts[i], assignment.unitType); + } } catch (final RuntimeException e) { if (failure == null) { failure = e; @@ -432,7 +442,10 @@ private RuntimeException triggerApplicableHoldoutExposures(final Experiment[] ho } private void triggerHoldoutExposure(final Experiment holdoutExperiment, final String unitType) { - final Assignment holdoutAssignment = getHoldoutAssignment(holdoutExperiment, unitType); + triggerHoldoutExposure(getHoldoutAssignment(holdoutExperiment, unitType)); + } + + private void triggerHoldoutExposure(final Assignment holdoutAssignment) { if ((holdoutAssignment != null) && holdoutAssignment.exposed.compareAndSet(false, true)) { enqueueExposure(holdoutAssignment); } @@ -819,11 +832,18 @@ private static class Assignment { boolean audienceMismatch; // Held out by a union of applicable holdouts: no exposure for this experiment, control // values only. `holdouts` is the resolved applicable list (by id+iteration identity, see - // holdoutSetMatches), used both to invalidate this cached assignment when coverage changes - // and to trigger each holdout's own exposure once this experiment is evaluated (see - // triggerExposure). + // holdoutSetMatches), used to invalidate this cached assignment when coverage changes. boolean suppressed; Experiment[] holdouts; + // The holdout Assignment objects the suppression decision above was made from, in the + // same order as `holdouts`, resolved via getHoldoutAssignment at the same instant as the + // decision (an entry is null only for an unconfigured unit type). triggerExposure fires + // exactly these rather than re-resolving `holdouts` against whatever data is live by the + // time exposure runs, so a refresh landing in between can never publish an exposure for a + // different epoch than the one the decision was made from. Null when no such snapshot was + // taken (the override path, which never decides suppression from holdouts); the trigger + // falls back to a live resolution of `holdouts` in that case. + Assignment[] holdoutAssignments; Map variables = Collections.emptyMap(); final AtomicBoolean exposed = new AtomicBoolean(false); @@ -940,15 +960,23 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, // pinned HoldoutAssignment (see getHoldoutAssignment) rather than recomputed // from the live definition here, so suppression and the holdout's own // exposure always agree, even after a same-iteration seed/split refresh. - for (final Experiment holdout : experiment.holdouts) { - final Assignment holdoutAssignment = Context.this.getHoldoutAssignment(holdout, - unitType); + // Every applicable holdout is resolved - the loop never stops at the first + // one that suppresses - because each one still owes its own exposure, and the + // resolved objects are pinned below (Assignment.holdoutAssignments) so + // triggerExposure fires exactly this decision's snapshot rather than + // re-resolving `holdouts` against data that may have moved to a different + // iteration by the time exposure runs. + final Assignment[] holdoutAssignments = new Assignment[experiment.holdouts.length]; + for (int i = 0; i < experiment.holdouts.length; ++i) { + final Assignment holdoutAssignment = Context.this + .getHoldoutAssignment(experiment.holdouts[i], unitType); + holdoutAssignments[i] = holdoutAssignment; if ((holdoutAssignment != null) && isHeldOutBy(holdoutAssignment.variant, holdoutAssignment.armCount, experiment.data.fullOnVariant)) { suppressed = true; - break; } } + assignment.holdoutAssignments = holdoutAssignments; } assignment.suppressed = suppressed; @@ -1068,8 +1096,7 @@ private Assignment getVariableAssignment(final String key, final boolean peek) { for (final ContextExperiment experimentVariables : keyExperimentVariables) { final Assignment assignment = getAssignment(experimentVariables.data.name); if (!peek) { - final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment.holdouts, - assignment.unitType); + final RuntimeException holdoutFailure = triggerApplicableHoldoutExposures(assignment); if (failure == null) { failure = holdoutFailure; } diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 4cddd8b..465e55d 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -5,6 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.verify; @@ -1017,6 +1018,54 @@ void experimentsEvaluatedBeforeAndAfterSameIterationArityChangeAgreeOnSuppressio assertEquals(NORMAL_VARIANT, context.getTreatment("exp_arity_consistency_pre")); } + // A refresh landing between the suppression decision and the exposure trigger must not let + // the published event mix two different holdout epochs. H@iteration1's arm does not suppress + // E; getTreatment(E) decides E's normal variant and queues E's own exposure. The event logger + // runs synchronously inside that enqueue (after E is queued, before the holdout trigger loop + // executes) and installs a refresh that bumps H to iteration2 with an arm that WOULD suppress + // E. The published event must still carry E's ordinary exposure together with H@iteration1's + // arm - the pair the decision was actually made from - never H@iteration2's arm. + // + // Fails against pre-change code: triggerApplicableHoldoutExposures re-resolves + // `assignment.holdouts` via getHoldoutAssignment at trigger time rather than firing a pinned + // snapshot, so by the time the trigger loop runs, getHoldoutAssignment sees the refreshed + // (iteration2) holdout definition, finds the iteration1 cache entry stale, recomputes against + // iteration2's arm (suppressing), and publishes E's ordinary exposure alongside a + // contradictory iteration2 variant-0 holdout exposure instead of iteration1's variant-1 one. + @Test + void triggerFiresHoldoutExposureFromDecisionEpochDespiteRefreshRacingBetweenDecisionAndTrigger() { + final Experiment holdoutIteration1 = newHoldout(11, "holdout_h", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Experiment experiment = coveredBy(newExperiment(1, "exp_epoch_race"), 11); + + final Experiment holdoutIteration2 = newHoldout(11, "holdout_h", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO); + holdoutIteration2.iteration = 2; + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_epoch_race"), 11); + when(dataProvider.getContextData()).thenReturn(CompletableFuture + .completedFuture(contextDataOf(new Experiment[]{holdoutIteration2}, refreshedExperiment))); + + final Context context = createReadyContext( + contextDataOf(new Experiment[]{holdoutIteration1}, experiment)); + + doAnswer(invocation -> { + final Object data = invocation.getArgument(2); + if ((data instanceof Exposure) && (((Exposure) data).id == 1)) { + context.refreshAsync().join(); + } + return null; + }).when(eventLogger).handleEvent(any(), any(), any()); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_epoch_race")); // not suppressed under iteration1 + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_epoch_race", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(11, "holdout_h", 1)); // iteration1's arm, not iteration2's + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + // --- Cross-SDK parity vectors ----------------------------------------------------------- // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the From 0f64b815c543cd402918a7bf20f5de6efe3ce26a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 13:45:40 +0000 Subject: [PATCH 37/49] fix: schedule the queued exposure even when the event logger throws enqueueExposure appends the exposure and increments pendingCount before invoking the ContextEventLogger callback, so a throwing logger left setTimeout() unreached and the queued exposure with no path to a flush. setTimeout() now runs in a finally around the callback; the callback's exception still propagates to the caller. --- .../main/java/com/absmartly/sdk/Context.java | 11 +++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 30 +++++++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index e0f19a2..d7bf54a 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -479,9 +479,14 @@ private void enqueueExposure(final Assignment assignment) { eventLock_.unlock(); } - logEvent(ContextEventLogger.EventType.Exposure, exposure); - - setTimeout(); + try { + logEvent(ContextEventLogger.EventType.Exposure, exposure); + } finally { + // Scheduled even if the callback above throws: the exposure is already appended and + // counted, so a skipped schedule would leave it queued with nothing left to flush it + // (setTimeout is idempotent, so scheduling here is never a duplicate concern). + setTimeout(); + } } public int peekTreatment(@Nonnull final String experimentName) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 465e55d..5eef3f0 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1277,6 +1277,36 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { assertEquals(2, context.getPendingCount()); } + // Regression test: a throwing logger for a suppressed experiment's ONLY holdout exposure must + // still schedule a flush. There is no ordinary exposure to fall back on for scheduling (the + // covered experiment is suppressed), so enqueueExposure's own setTimeout() call is the only + // thing that can ever flush this queued exposure - if it is skipped because logEvent threw, + // the exposure is stranded in the queue for the lifetime of the context. The exception from + // the logger must still propagate to the caller. + // + // Fails against pre-change code: setTimeout() is called after logEvent() with no finally, so + // the throw from the holdout exposure's logEvent call skips setTimeout() entirely - the + // exposure is queued (pendingCount == 1) but scheduler.schedule is never invoked. + @Test + void throwingLoggerOnSuppressedExperimentsOnlyHoldoutStillSchedulesFlush() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), any(Exposure.class)); + + final Experiment experiment = coveredBy(newExperiment(1, "exp_stranded_flush"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO)}, experiment)); + + when(scheduler.schedule((Runnable) any(), eq(100L), eq(TimeUnit.MILLISECONDS))) + .thenReturn(mock(java.util.concurrent.ScheduledFuture.class)); + + assertThrows(RuntimeException.class, () -> context.getTreatment("exp_stranded_flush")); + + // the holdout's exposure was queued (this experiment has no exposure of its own - it is + // suppressed) and a flush was scheduled for it despite the throw. + assertEquals(1, context.getPendingCount()); + verify(scheduler, Mockito.timeout(5000).times(1)).schedule((Runnable) any(), eq(100L), + eq(TimeUnit.MILLISECONDS)); + } + // Malformed holdouts are omitted from the id index; references to omitted entries must not // affect normal experiment assignment. @Test From c6a0a5a6995f97f68e2812e0ee797987a8103e85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 20:52:14 +0000 Subject: [PATCH 38/49] fix: evict null holdout snapshots on setUnit so a late unit is not lost getAssignment's holdoutAssignments snapshot (55b87fe) pins whatever getHoldoutAssignment returns for each applicable holdout, including null when the holdout's unit type is not yet installed (Context.java ~1208: units_.get(unitType) returns null before the uid is set). A full-on covered experiment does not need the unit to produce a variant - the uid is only read in the fullOnVariant == 0 branch - so the experiment resolves normally while the holdout silently could not be asked, and the null gets pinned with suppressed=false. That is the invalidation axis here: the input became complete, not that the data changed, so the existing matches-based cache checks (experimentMatches/holdoutSetMatches, comparing (id, iteration)) correctly report no change and never catch it. triggerExposure fires the pinned snapshot directly and triggerHoldoutExposure(Assignment) skips a null entry outright, so once setUnit() installs the unit, the holdout exposure is lost for the life of the context - setUnit only wrote units_ and never touched the cache. setUnit now evicts, targeted by the covered experiment's unit type, any cached Assignment whose holdoutAssignments snapshot holds a null entry for that unit. Eviction forces getAssignment to recompute suppression and every owed exposure from one coherent decision made with the complete unit set, rather than re-resolving the null in place and firing a freshly-resolved holdout exposure next to a decision that was made without it - which would publish "held out" alongside "participated" for the same unit. Two defects are fixed relative to the rejected 6ef3545: 1. Wrong predicate field. The null entry is produced by getHoldoutAssignment(holdout, assignment.unitType) - the covered experiment's unit type - never by the referenced holdout's own declared unitType, which nothing requires to match. The eviction predicate now compares against assignment.unitType (the value resolution actually used), invoking equals on the non-null setUnit argument since Experiment.unitType is nullable. 2. Eviction resetting exposure state. Removing a cached Assignment discards its exposed AtomicBoolean. A full-on covered experiment is assignable without the unit, so its exposure can already be queued before the unit arrives; evicting that Assignment let the next getTreatment() build a fresh one with exposed==false, causing a duplicate (or contradictory, if the resolved holdout now suppresses) exposure for the same unit. Eviction now skips any assignment whose exposed flag is already set - once participation has been published for a unit, retroactively holding it out is wrong, and leaving one degraded record alone is better than compounding it with a contradictory second one. publish() builds event.units from the live units_ map at publish time regardless, so an exposure queued before setUnit may already carry the late unit either way. Adds selectivity tests pinning the shape of the fix rather than just the original bug: already-exposed assignments survive eviction (own and unrelated unit type), unaffected cache entries are left untouched, eviction is keyed off the covered experiment's unit type rather than the holdout's declared one, a late-resolved suppressing holdout produces a coherent recompute (kills the "re-resolve null at trigger time" alternative), multiple holdouts with different declared unit types on one experiment evict together, and setUnit with no cached assignment is a no-op that doesn't throw. Known residual, filed separately and not addressed here: getTreatment calls getAssignment(...) then triggerExposure(...), so a thread can hold a stale Assignment object across another thread's eviction and expose it after a replacement was exposed; exposed flags are per-object so locking the map does not prevent it. --- .../main/java/com/absmartly/sdk/Context.java | 62 +++++ .../com/absmartly/sdk/ContextHoldoutTest.java | 225 ++++++++++++++++++ 2 files changed, 287 insertions(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index d7bf54a..60e5a96 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -299,11 +299,73 @@ public void setUnit(@Nonnull final String unitType, @Nonnull final String uid) { } units_.put(unitType, trimmed); + + invalidateAssignmentsPinnedWithMissingUnit(unitType); } finally { writeLock.unlock(); } } + // A cached Assignment's holdoutAssignments[i] is null exactly when unitType was absent at the + // time getAssignment() resolved holdout i (see the snapshot loop below getHoldoutAssignment). + // The trigger loop pins that snapshot and skips a null entry rather than re-resolving it live + // (triggerHoldoutExposure(Assignment)), so once uid becomes available the pinned null would + // never fire - permanently losing an exposure that is now perfectly evaluable. Re-resolving + // the null in place instead of evicting is wrong: the suppression decision on the cached + // Assignment was computed while the unit was missing (holdout treated as not-suppressing), so + // firing a freshly-resolved holdout exposure next to it would publish "held out" alongside + // "participated" for the same unit - an incoherent pair that reintroduces the exact + // decision/exposure mismatch the pinned snapshot exists to prevent. Evicting the cache entry + // instead forces getAssignment() to recompute suppression and every exposure from one + // coherent decision made with the now-complete unit set. + // + // The null entry was produced by getHoldoutAssignment(holdout, assignment.unitType) - the + // COVERED experiment's unit type, i.e. the value resolved into assignment.unitType and passed + // as `unitType` to every getHoldoutAssignment call in that snapshot loop. A referenced + // holdout's own declared unitType (holdout.unitType) is never read for that lookup, and + // nothing requires it to match the covered experiment's, so comparing against it instead is + // the wrong predicate: it can miss the eviction this unit installation actually unblocks, or + // evict entries this installation has nothing to do with. Comparing against assignment.unitType + // (equals invoked on the non-null setUnit argument since Experiment.unitType is nullable) is + // the value resolution actually used. + // + // Only entries whose null snapshot belongs to an assignment requiring exactly this unitType + // are evicted - unrelated cached assignments (whose holdouts already had their unit, or that + // hold no holdouts at all) are left untouched, so an unrelated setUnit() call never discards + // unaffected cache state. + // + // An assignment already exposed is left alone even if it holds a null entry: `exposed` is not + // re-created by eviction, so removing an exposed Assignment would let the next getTreatment() + // build a fresh one with exposed==false, re-publishing (or, if the newly-resolved holdout now + // suppresses, contradicting) an exposure already recorded for that unit. publish() reads + // event.units from the live units_ map at publish time, so an exposure queued before this + // setUnit call may already carry the late unit regardless - the record is degraded either + // way, and this guard exists only to avoid compounding that with a second, contradictory one, + // not to pretend the degraded record is correct. + // + // The recomputed Assignment may legitimately choose a different variant than an earlier + // peekTreatment() returned for the same experiment: that peek ran with an incomplete unit set, + // and peekTreatment() never publishes an exposure, so there is nothing stale to reconcile. + private void invalidateAssignmentsPinnedWithMissingUnit(final String unitType) { + final Iterator it = assignmentCache_.values().iterator(); + while (it.hasNext()) { + final Assignment assignment = it.next(); + if (assignment.exposed.get()) { + continue; + } + + final Assignment[] holdoutAssignments = assignment.holdoutAssignments; + if ((holdoutAssignments != null) && unitType.equals(assignment.unitType)) { + for (final Assignment holdoutAssignment : holdoutAssignments) { + if (holdoutAssignment == null) { + it.remove(); + break; + } + } + } + } + } + public Map getUnits() { final ReentrantReadWriteLock.ReadLock readLock = contextLock_.readLock(); try { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 5eef3f0..eaf770d 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1066,6 +1066,231 @@ void triggerFiresHoldoutExposureFromDecisionEpochDespiteRefreshRacingBetweenDeci verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // Reproduces the null-snapshot regression: a ready context with no unit installed peeks a + // covered full-on experiment (peekTreatment never requires the covered unit type - only the + // holdout resolution does), pinning a null holdoutAssignments entry into the cached + // Assignment because "user_id" was absent. setUnit("user_id", ...) must evict that cache + // entry so getTreatment recomputes the decision with the now-present unit, instead of + // reusing the pinned null and silently skipping the holdout's exposure forever. + // + // Fails against pre-fix code: setUnit only writes units_, never touches assignmentCache_, so + // getTreatment's cache-hit path (experimentMatches: same iteration, same holdouts array) + // reuses the stale Assignment. triggerApplicableHoldoutExposures then fires the pinned + // snapshot, sees holdoutAssignments[0] == null, and triggerHoldoutExposure(Assignment) skips + // a null argument outright - only the experiment's own exposure is queued, ever. + @Test + void setUnitInvalidatesNullHoldoutSnapshotSoTheHoldoutExposureIsNotLostForever() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_null_snapshot_fullon", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out for UID once resolvable + + final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // peekTreatment resolves the full-on variant without ever needing coveredUnitType; the + // holdout resolution against the missing unit yields a null snapshot entry, pinned into + // the cached Assignment without suppressing (a null entry never suppresses). + assertEquals(2, context.peekTreatment("exp_null_snapshot_fullon")); + + context.setUnit(coveredUnitType, UID); + + // recomputed from a coherent decision now that the unit is present: still fullOn=2 (this + // holdout's arm does not hold it out), but this time both exposures are owed. + assertEquals(2, context.getTreatment("exp_null_snapshot_fullon")); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_null_snapshot_fullon", coveredUnitType, 2, clock.millis(), true, true, false, + true, false, false), + holdoutExposure(coveredUnitType, 11, "holdout_user_id", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Selectivity for Defect 2 (High): eviction must not discard an already-exposed assignment. + // A full-on covered experiment is assignable without the unit, so getTreatment (not + // peekTreatment) queues its own exposure immediately while the holdout's snapshot entry is + // still null. setUnit must leave this exposed Assignment in place; a subsequent getTreatment + // must not re-queue the experiment's exposure a second time. + // + // Fails without the `!assignment.exposed.get()` guard: eviction removes the exposed + // Assignment, the next getTreatment builds a fresh one with exposed==false, and the + // experiment's exposure is published twice for the same unit. + @Test + void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_exposed_before_unit", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Context context = createReadyContext(ContextConfig.create(), + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.getTreatment("exp_exposed_before_unit")); + assertEquals(1, context.getPendingCount()); + + context.setUnit(coveredUnitType, UID); + + assertEquals(2, context.getTreatment("exp_exposed_before_unit")); + assertEquals(1, context.getPendingCount()); // unchanged: no duplicate exposure + } + + // Selectivity: an unrelated unit type must never touch cache entries at all, exposed or not. + // Fails under `assignmentCache_.clear()`, which wipes every entry regardless of unit type. + @Test + void setUnitForUnrelatedUnitTypeDoesNotEvictExposedAssignment() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_unrelated_unit"), 11); + final Context context = createReadyContext(contextDataOf( + new Experiment[]{newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO)}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unrelated_unit")); + assertEquals(2, context.getPendingCount()); // experiment's own exposure + holdout's + + context.setUnit("other_unit_type", "some-other-uid"); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unrelated_unit")); + assertEquals(2, context.getPendingCount()); // unchanged: no duplicate exposure + } + + // Selectivity: with two cached experiments, only the one whose snapshot holds a null entry + // for the just-installed unit type is evicted; the other, already resolved and exposed + // against its own (already-present) unit type, is left untouched. + @Test + void setUnitEvictsOnlyTheAffectedEntryAndLeavesTheOtherUntouched() { + final String lateUnitType = "user_id"; + final Experiment lateExperiment = coveredBy( + newExperiment(1, "exp_late_unit", lateUnitType, 2), 11); + final Experiment lateHoldout = newHoldout(11, "holdout_late", lateUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Experiment presentExperiment = coveredBy(newExperiment(2, "exp_present_unit"), 12); + final Experiment presentHoldout = newHoldout(12, "holdout_present", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + + final Context context = createReadyContext(UID, // installs UNIT_TYPE only + contextDataOf(new Experiment[]{lateHoldout, presentHoldout}, lateExperiment, presentExperiment)); + + assertEquals(2, context.peekTreatment("exp_late_unit")); // null snapshot, not yet exposed + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_present_unit")); // resolved and exposed + assertEquals(2, context.getPendingCount()); // exp_present_unit + holdout_present + + context.setUnit(lateUnitType, UID); + + // the affected entry recomputes and owes both of its exposures. + assertEquals(2, context.getTreatment("exp_late_unit")); + assertEquals(4, context.getPendingCount()); + + // the unaffected entry must not be re-triggered by the eviction pass or by this + // unrelated getTreatment call. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_present_unit")); + assertEquals(4, context.getPendingCount()); + } + + // Defect 1 (Medium): the predicate must compare against assignment.unitType - the value + // getHoldoutAssignment was actually called with (experiment.data.unitType) - not the + // referenced holdout's own declared unitType, which nothing requires to match. + // + // Fails against `unitType.equals(assignment.holdouts[i].unitType)`: the covered experiment's + // unitType is "A", the holdout declares "B", and the null snapshot was produced by "A" being + // absent. setUnit("A", ...) must evict, but the old predicate compares "A" against the + // holdout's declared "B" and never does. + @Test + void setUnitEvictsUsingTheCoveredExperimentsUnitTypeNotTheHoldoutsDeclaredUnitType() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_mismatched_unit_type", "A", 2), 11); + final Experiment holdout = newHoldout(11, "holdout_declares_b", "B", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final Context context = createReadyContext(ContextConfig.create(), // neither "A" nor "B" installed + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.peekTreatment("exp_mismatched_unit_type")); // null snapshot: "A" is absent + + context.setUnit("A", UID); + + assertEquals(2, context.getTreatment("exp_mismatched_unit_type")); + assertEquals(2, context.getPendingCount()); // both exposures owed: eviction happened + } + + // Kills the "re-resolve the null live at trigger time" mutant: the late-resolved holdout's + // arm actually suppresses this unit, so a coherent recomputation must flip the experiment + // from its pinned fullOn=2/unsuppressed verdict to control (0) and publish only the holdout's + // exposure. Re-resolving the null in place instead of recomputing the whole decision would + // leave variant=2/unsuppressed as-is and publish a contradictory "held out" holdout exposure + // alongside a "participated" experiment exposure for the same unit. + @Test + void setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_null_snapshot_suppresses", coveredUnitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_user_id_suppress", coveredUnitType, HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContext(ContextConfig.create(), + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(2, context.peekTreatment("exp_null_snapshot_suppresses")); // null snapshot never suppresses + + context.setUnit(coveredUnitType, UID); + + assertEquals(0, context.getTreatment("exp_null_snapshot_suppresses")); // now suppressed -> control + assertEquals(1, context.getPendingCount()); // holdout exposure only + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + holdoutExposure(coveredUnitType, 11, "holdout_user_id_suppress", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Multiple holdouts applicable to one covered experiment, each declaring a different (and + // irrelevant, per Defect 1) unitType, guard against wrong-index matching or stopping the + // eviction scan at the wrong entry. Every applicable holdout is resolved against the SAME + // value - the covered experiment's own unitType - so all three snapshot entries are null + // together, and setUnit must evict once and let all three, plus the experiment's own, fire. + @Test + void setUnitEvictsAssignmentWithMultipleHoldoutsDeclaringDifferentUnitTypes() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_multi_holdout_unit_types", coveredUnitType, 2), 11, 12, 13); + final Experiment holdoutA = newHoldout(11, "holdout_11", "type_x", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + final Experiment holdoutB = newHoldout(12, "holdout_12", "type_y", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + final Experiment holdoutC = newHoldout(13, "holdout_13", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); + + final Context context = createReadyContext(ContextConfig.create(), + contextDataOf(new Experiment[]{holdoutA, holdoutB, holdoutC}, experiment)); + + assertEquals(2, context.peekTreatment("exp_multi_holdout_unit_types")); + + context.setUnit(coveredUnitType, UID); + + assertEquals(2, context.getTreatment("exp_multi_holdout_unit_types")); + assertEquals(4, context.getPendingCount()); // experiment + 3 holdout exposures + } + + // setUnit must be a no-op with respect to the assignment cache when nothing has been cached + // yet for this context - no exception, and the subsequent assignment behaves normally. + @Test + void setUnitWithNoCachedAssignmentDoesNotThrowAndSubsequentAssignmentIsNormal() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_no_prior_assignment"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final Context context = createReadyContext(ContextConfig.create(), + contextDataOf(new Experiment[]{holdout}, experiment)); + + context.setUnit(UNIT_TYPE, UID); // nothing cached yet - must not throw + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_no_prior_assignment")); + } + // --- Cross-SDK parity vectors ----------------------------------------------------------- // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the From dc2afeb6f29ae0661067fb2c26e690348f56d71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 22:57:13 +0000 Subject: [PATCH 39/49] test: close the surviving null-entry eviction mutant on setUnit Mutant "evict whenever holdoutAssignments != null" (dropping the check that some entry is null) survived 287/287 against c6a0a5a. It is only observable when a fully-resolved, unexposed cache entry is wrongly discarded by a redundant setUnit call for the same unit type: setUnitRedundantCallDoesNotEvictFullyResolvedUnexposedHoldoutSnapshot pins the entry's audience-mismatch verdict across a redundant setUnit and an attribute change that experimentMatches ignores; a wrongly-evicted entry would recompute under the now-matching attribute and flip the variant. triggerApplicableHoldoutExposuresNeverLiveResolvesAPinnedNullEntryForAn- AlreadyExposedAssignment covers a mutation the earlier fix attempt already guards against in the common case (setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses), but not in the one setUnit deliberately leaves unevicted: an already-exposed assignment's pinned null survives setUnit forever (Defect 2 guard), and can still reach triggerApplicableHoldoutExposures via the variable-key path, which re-triggers every candidate's holdouts unconditionally on every call regardless of that candidate's own exposed state. Also adds: peekVariableValue/getVariableValue coverage for the late-unit path with a suppressing late arm distinguishing recompute from live patch-up; override and custom-assignment interaction coverage (overrides never take a snapshot and rely on live resolution at exposure time; custom assignments take the ordinary snapshot path and are evicted the same way); and a redundant setUnit(same unit type, same uid) no-op check. --- .../com/absmartly/sdk/ContextHoldoutTest.java | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index eaf770d..0f1c90d 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1247,6 +1247,47 @@ void setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + // Kills a narrower variant of the "re-resolve the null live at trigger time" mutant that + // setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses cannot reach: once + // an Assignment is already exposed, setUnit's `!exposed` guard deliberately leaves its null + // snapshot entry in the cache forever (Defect 2), so a null pinned entry can still reach + // triggerApplicableHoldoutExposures on a LATER evaluation of the SAME already-exposed + // assignment - specifically via the variable-key path, which re-triggers every candidate's + // holdouts unconditionally on each call, independent of that candidate's own exposed state. + // Correct code passes the pinned null straight to triggerHoldoutExposure(Assignment), which + // no-ops on null every time, so the holdout is never fired for this unit's life once the + // snapshot went stale exposed. The mutant instead re-resolves live once the unit is present + // and fires a fresh, contradictory "held out" exposure next to the "participated" one already + // published. + @Test + void triggerApplicableHoldoutExposuresNeverLiveResolvesAPinnedNullEntryForAnAlreadyExposedAssignment() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_a_mutant_pinned_null", coveredUnitType, 1), 11); + experiment.variants[1].config = "{\"my_var\":\"value_from_variant\"}"; + final Experiment holdout = newHoldout(11, "holdout_a_mutant_pinned_null", coveredUnitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // fullOn resolves and exposes without ever needing coveredUnitType; the holdout's null + // snapshot entry never suppresses and, being null both here (unit absent) and under the + // mutant's live resolution (uid still null in units_), never fires either. + assertEquals(1, context.getTreatment("exp_a_mutant_pinned_null")); + assertEquals(1, context.getPendingCount()); // own exposure only + + context.setUnit(coveredUnitType, UID); // exposed guard: cache entry survives with pinned null + + // re-evaluating the same already-exposed assignment via the variable-key path, which + // unconditionally re-triggers every candidate's applicable holdouts on every call. + assertEquals("value_from_variant", context.getVariableValue("my_var", "default")); + + // correct code: pinned[0] is still null, triggerHoldoutExposure(null) no-ops - no second + // exposure, ever, for this permanently-degraded assignment. + assertEquals(1, context.getPendingCount()); + } + // Multiple holdouts applicable to one covered experiment, each declaring a different (and // irrelevant, per Defect 1) unitType, guard against wrong-index matching or stopping the // eviction scan at the wrong entry. Every applicable holdout is resolved against the SAME @@ -1291,6 +1332,185 @@ void setUnitWithNoCachedAssignmentDoesNotThrowAndSubsequentAssignmentIsNormal() assertEquals(NORMAL_VARIANT, context.getTreatment("exp_no_prior_assignment")); } + // A redundant setUnit call for a unit type that is already installed with the same uid must + // be a pure no-op: no exception, no re-evaluation, no duplicate exposure. + @Test + void setUnitCalledTwiceWithSameUidIsANoOpAndDoesNotThrow() { + final Experiment experiment = coveredBy(newExperiment(1, "exp_set_unit_twice"), 11); + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // not held out + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_set_unit_twice")); + assertEquals(2, context.getPendingCount()); // own exposure + holdout's + + context.setUnit(UNIT_TYPE, UID); // same unit type, same uid: must not throw + + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_set_unit_twice")); + assertEquals(2, context.getPendingCount()); // unchanged: no duplicate exposure, cache untouched + } + + // Kills the "evict on holdoutAssignments != null alone" mutant (dropping the check that some + // entry is actually null): a cached, unexposed assignment whose holdout snapshot is fully + // resolved (no null entries, because the unit was already present when it was computed) must + // survive a redundant setUnit call for that same unit type. The distinguishing signal is an + // attribute change made between the peek and the redundant setUnit call: experimentMatches + // never considers attributes, so a retained cache entry keeps the audience verdict (and thus + // the variant) it was computed with, while a wrongly-evicted entry would recompute against + // the now-matching attribute and land on a different variant entirely. + // + // Fails against `(holdoutAssignments != null) && unitType.equals(assignment.unitType)` alone: + // the array is non-null (one resolved holdout entry) and the unit type matches, so the entry + // is evicted even though it holds no null entry, and the redundant setUnit call silently + // re-decides the experiment. + @Test + void setUnitRedundantCallDoesNotEvictFullyResolvedUnexposedHoldoutSnapshot() { + final String audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; + + final Experiment experiment = coveredBy(newExperiment(1, "exp_redundant_set_unit"), 11); + experiment.audienceStrict = true; + experiment.audience = audience; + final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // not held out + + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + context.setAttribute("age", 5); // mismatches: forces control regardless of holdout + + // resolved with the unit already present: holdoutAssignments holds no null entry. + assertEquals(0, context.peekTreatment("exp_redundant_set_unit")); // audience mismatch -> control + assertEquals(0, context.getPendingCount()); // peek never exposes + + context.setAttribute("age", 25); // now matches; experimentMatches ignores attributes though + context.setUnit(UNIT_TYPE, UID); // redundant: same unit type, same uid already installed + + // still the pinned, audience-mismatched decision: a wrongly-evicted entry would recompute + // under the now-matching attribute and land on NORMAL_VARIANT instead. + assertEquals(0, context.getTreatment("exp_redundant_set_unit")); + assertEquals(2, context.getPendingCount()); // own exposure (audience-mismatch) + holdout's + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(UID, + new Exposure(1, "exp_redundant_set_unit", UNIT_TYPE, 0, clock.millis(), false, true, false, false, + false, true), + holdoutExposure(11, "holdout_a", 1)); + expected.attributes = new Attribute[]{new Attribute("age", 5, clock.millis()), + new Attribute("age", 25, clock.millis())}; + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 3: peekVariableValue populates the same cache as getAssignment/getTreatment, and + // getVariableValue has its own per-candidate holdout triggering. A late-resolved SUPPRESSING + // holdout must flip the recomputed decision (fullOn -> held out), which is only observable if + // the cache entry was actually recomputed rather than live-patched: a live patch-up would + // leave the fullOn variant (and its variable value) untouched and merely append a + // contradictory holdout exposure next to it. + @Test + void peekVariableValueThenSetUnitThenGetVariableValueRecomputesUnderLateSuppressingHoldout() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_var_late_unit_suppress", coveredUnitType, 1), 11); + experiment.variants[1].config = "{\"my_var\":\"value_from_variant\"}"; + final Experiment holdout = newHoldout(11, "holdout_var_late", coveredUnitType, HOLDOUT_A_SEED_HI, + HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable + + final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // fullOn resolves without needing coveredUnitType; the holdout's null snapshot entry + // never suppresses, so the fullOn variant's variable value wins. + assertEquals("value_from_variant", context.peekVariableValue("my_var", "default")); + assertEquals(0, context.getPendingCount()); + + context.setUnit(coveredUnitType, UID); + + // recomputed from a coherent decision: the now-resolvable holdout holds UID out, so the + // experiment is suppressed and control values (no "my_var" key) are returned. + assertEquals("default", context.getVariableValue("my_var", "default")); + assertEquals(1, context.getPendingCount()); // holdout's own exposure only + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + holdoutExposure(coveredUnitType, 11, "holdout_var_late", 0)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 4a: overrides never take a holdoutAssignments snapshot (the override branch in + // getAssignment never calls getHoldoutAssignment), so invalidateAssignmentsPinnedWithMissingUnit + // leaves an overridden assignment untouched regardless of a late setUnit - by design, it + // relies on triggerApplicableHoldoutExposures' live-resolution fallback (pinned == null) at + // the moment of exposure instead. Pinning this: the unit is absent when the override is + // peeked, arrives via setUnit before the assignment is ever exposed, and the holdout must + // still be correctly resolved (live, against the now-present unit) when exposure finally + // fires. + @Test + void setUnitDoesNotEvictOverrideAssignmentAndHoldoutResolvesLiveAtExposureTime() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_override_late_unit", coveredUnitType, 0), 11); + final Experiment holdout = newHoldout(11, "holdout_override_late", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out once resolvable + + final ContextConfig config = ContextConfig.create().setOverride("exp_override_late_unit", 3); + final Context context = createReadyContext(config, // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + assertEquals(3, context.peekTreatment("exp_override_late_unit")); // override wins; not exposed + + context.setUnit(coveredUnitType, UID); // no snapshot to evict; override assignment untouched + + assertEquals(3, context.getTreatment("exp_override_late_unit")); // override still wins + assertEquals(2, context.getPendingCount()); // override's own exposure + holdout's (live-resolved) + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_override_late_unit", coveredUnitType, 3, clock.millis(), false, true, true, + false, false, false), + holdoutExposure(coveredUnitType, 11, "holdout_override_late", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + + // Task 4b: a custom assignment takes the ordinary snapshot path (getAssignment's non-override + // branch), so it is subject to the same null-snapshot eviction as any other covered + // experiment. The custom assignment can only apply once the experiment's own unit is present + // (it is read inside the traffic-eligibility branch, which is itself gated on the unit), so + // initially it cannot apply at all; after the late setUnit forces a coherent recompute, the + // custom assignment applies normally. + @Test + void setUnitEvictsNullSnapshotForCustomAssignmentAndCustomAppliesAfterRecompute() { + final String coveredUnitType = "user_id"; + final Experiment experiment = coveredBy( + newExperiment(1, "exp_custom_late_unit", coveredUnitType, 0), 11); + final Experiment holdout = newHoldout(11, "holdout_custom_late", coveredUnitType, HOLDOUT_B_SEED_HI, + HOLDOUT_B_SEED_LO, "full"); // not held out once resolvable + + final ContextConfig config = ContextConfig.create().setCustomAssignment("exp_custom_late_unit", 3); + final Context context = createReadyContext(config, // no unit installed yet + contextDataOf(new Experiment[]{holdout}, experiment)); + + // the custom assignment is read only inside the uid-present branch, so with the unit + // absent it cannot apply at all: control values. + assertEquals(0, context.peekTreatment("exp_custom_late_unit")); + + context.setUnit(coveredUnitType, UID); // evicts the null-snapshot cache entry + + assertEquals(3, context.getTreatment("exp_custom_late_unit")); // custom applies now + assertEquals(2, context.getPendingCount()); // own exposure + holdout's + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final PublishEvent expected = publishedEvent(coveredUnitType, UID, + new Exposure(1, "exp_custom_late_unit", coveredUnitType, 3, clock.millis(), true, true, false, false, + true, false), + holdoutExposure(coveredUnitType, 11, "holdout_custom_late", 1)); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); + } + // --- Cross-SDK parity vectors ----------------------------------------------------------- // Verdicts below were computed offline against the SDK's own MD5 -> base64url-unpadded -> // murmur3_32 pipeline (VariantAssigner/UnitHasher, unmodified) and independently against the From c3bd8556998f2ec10e0f551f31f9a71a8d431e50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 23:16:51 +0000 Subject: [PATCH 40/49] refactor: simplify late-unit invalidation coverage --- .../main/java/com/absmartly/sdk/Context.java | 46 ++++--------------- .../com/absmartly/sdk/ContextHoldoutTest.java | 20 ++++---- 2 files changed, 20 insertions(+), 46 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 60e5a96..68f39a4 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -306,46 +306,16 @@ public void setUnit(@Nonnull final String unitType, @Nonnull final String uid) { } } - // A cached Assignment's holdoutAssignments[i] is null exactly when unitType was absent at the - // time getAssignment() resolved holdout i (see the snapshot loop below getHoldoutAssignment). - // The trigger loop pins that snapshot and skips a null entry rather than re-resolving it live - // (triggerHoldoutExposure(Assignment)), so once uid becomes available the pinned null would - // never fire - permanently losing an exposure that is now perfectly evaluable. Re-resolving - // the null in place instead of evicting is wrong: the suppression decision on the cached - // Assignment was computed while the unit was missing (holdout treated as not-suppressing), so - // firing a freshly-resolved holdout exposure next to it would publish "held out" alongside - // "participated" for the same unit - an incoherent pair that reintroduces the exact - // decision/exposure mismatch the pinned snapshot exists to prevent. Evicting the cache entry - // instead forces getAssignment() to recompute suppression and every exposure from one - // coherent decision made with the now-complete unit set. + // holdoutAssignments pins the holdout decisions used to compute suppression. A null entry means + // the covered experiment's unit was unavailable; resolving it live later could publish a holdout + // verdict inconsistent with the cached experiment decision, so setUnit evicts the assignment and + // lets getAssignment recompute the decision and exposures together. // - // The null entry was produced by getHoldoutAssignment(holdout, assignment.unitType) - the - // COVERED experiment's unit type, i.e. the value resolved into assignment.unitType and passed - // as `unitType` to every getHoldoutAssignment call in that snapshot loop. A referenced - // holdout's own declared unitType (holdout.unitType) is never read for that lookup, and - // nothing requires it to match the covered experiment's, so comparing against it instead is - // the wrong predicate: it can miss the eviction this unit installation actually unblocks, or - // evict entries this installation has nothing to do with. Comparing against assignment.unitType - // (equals invoked on the non-null setUnit argument since Experiment.unitType is nullable) is - // the value resolution actually used. + // Holdouts in this snapshot are resolved using assignment.unitType, not each holdout's declared + // unitType, so only that unit installation invalidates a null entry. // - // Only entries whose null snapshot belongs to an assignment requiring exactly this unitType - // are evicted - unrelated cached assignments (whose holdouts already had their unit, or that - // hold no holdouts at all) are left untouched, so an unrelated setUnit() call never discards - // unaffected cache state. - // - // An assignment already exposed is left alone even if it holds a null entry: `exposed` is not - // re-created by eviction, so removing an exposed Assignment would let the next getTreatment() - // build a fresh one with exposed==false, re-publishing (or, if the newly-resolved holdout now - // suppresses, contradicting) an exposure already recorded for that unit. publish() reads - // event.units from the live units_ map at publish time, so an exposure queued before this - // setUnit call may already carry the late unit regardless - the record is degraded either - // way, and this guard exists only to avoid compounding that with a second, contradictory one, - // not to pretend the degraded record is correct. - // - // The recomputed Assignment may legitimately choose a different variant than an earlier - // peekTreatment() returned for the same experiment: that peek ran with an incomplete unit set, - // and peekTreatment() never publishes an exposure, so there is nothing stale to reconcile. + // Only unexposed assignments are evicted. Eviction creates a new exposed flag, so evicting an + // already-exposed assignment could publish a duplicate or contradictory experiment exposure. private void invalidateAssignmentsPinnedWithMissingUnit(final String unitType) { final Iterator it = assignmentCache_.values().iterator(); while (it.hasNext()) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 0f1c90d..ea7889a 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -102,6 +102,10 @@ Context createReadyContext(ContextConfig config, ContextData data) { eventHandler, eventLogger, variableParser, audienceMatcher); } + Context createReadyContextWithoutUnits(ContextData data) { + return createReadyContext(ContextConfig.create(), data); + } + static Experiment newExperiment(int id, String name) { return newExperiment(id, name, UNIT_TYPE, 0); } @@ -1086,7 +1090,7 @@ void setUnitInvalidatesNullHoldoutSnapshotSoTheHoldoutExposureIsNotLostForever() final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); // not held out for UID once resolvable - final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + final Context context = createReadyContextWithoutUnits( // no unit installed yet contextDataOf(new Experiment[]{holdout}, experiment)); // peekTreatment resolves the full-on variant without ever needing coveredUnitType; the @@ -1128,7 +1132,7 @@ void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() { final Experiment holdout = newHoldout(11, "holdout_user_id", coveredUnitType, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); - final Context context = createReadyContext(ContextConfig.create(), + final Context context = createReadyContextWithoutUnits( contextDataOf(new Experiment[]{holdout}, experiment)); assertEquals(2, context.getTreatment("exp_exposed_before_unit")); @@ -1204,7 +1208,7 @@ void setUnitEvictsUsingTheCoveredExperimentsUnitTypeNotTheHoldoutsDeclaredUnitTy final Experiment holdout = newHoldout(11, "holdout_declares_b", "B", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); - final Context context = createReadyContext(ContextConfig.create(), // neither "A" nor "B" installed + final Context context = createReadyContextWithoutUnits( // neither "A" nor "B" installed contextDataOf(new Experiment[]{holdout}, experiment)); assertEquals(2, context.peekTreatment("exp_mismatched_unit_type")); // null snapshot: "A" is absent @@ -1229,7 +1233,7 @@ void setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses() { final Experiment holdout = newHoldout(11, "holdout_user_id_suppress", coveredUnitType, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable - final Context context = createReadyContext(ContextConfig.create(), + final Context context = createReadyContextWithoutUnits( contextDataOf(new Experiment[]{holdout}, experiment)); assertEquals(2, context.peekTreatment("exp_null_snapshot_suppresses")); // null snapshot never suppresses @@ -1268,7 +1272,7 @@ void triggerApplicableHoldoutExposuresNeverLiveResolvesAPinnedNullEntryForAnAlre final Experiment holdout = newHoldout(11, "holdout_a_mutant_pinned_null", coveredUnitType, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable - final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + final Context context = createReadyContextWithoutUnits( // no unit installed yet contextDataOf(new Experiment[]{holdout}, experiment)); // fullOn resolves and exposes without ever needing coveredUnitType; the holdout's null @@ -1305,7 +1309,7 @@ void setUnitEvictsAssignmentWithMultipleHoldoutsDeclaringDifferentUnitTypes() { final Experiment holdoutC = newHoldout(13, "holdout_13", coveredUnitType, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); - final Context context = createReadyContext(ContextConfig.create(), + final Context context = createReadyContextWithoutUnits( contextDataOf(new Experiment[]{holdoutA, holdoutB, holdoutC}, experiment)); assertEquals(2, context.peekTreatment("exp_multi_holdout_unit_types")); @@ -1324,7 +1328,7 @@ void setUnitWithNoCachedAssignmentDoesNotThrowAndSubsequentAssignmentIsNormal() final Experiment holdout = newHoldout(11, "holdout_a", UNIT_TYPE, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); - final Context context = createReadyContext(ContextConfig.create(), + final Context context = createReadyContextWithoutUnits( contextDataOf(new Experiment[]{holdout}, experiment)); context.setUnit(UNIT_TYPE, UID); // nothing cached yet - must not throw @@ -1414,7 +1418,7 @@ void peekVariableValueThenSetUnitThenGetVariableValueRecomputesUnderLateSuppress final Experiment holdout = newHoldout(11, "holdout_var_late", coveredUnitType, HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); // holds UID out once resolvable - final Context context = createReadyContext(ContextConfig.create(), // no unit installed yet + final Context context = createReadyContextWithoutUnits( // no unit installed yet contextDataOf(new Experiment[]{holdout}, experiment)); // fullOn resolves without needing coveredUnitType; the holdout's null snapshot entry From 971203264ea539ad3e07209d7e21ac3c3422c189 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Mon, 31 Aug 2026 23:19:39 +0000 Subject: [PATCH 41/49] docs: keep the degraded-record caveat on the exposed guard The condensed comment lost the least-obvious half of why unexposed-only eviction is right: publish() reads event.units from the live units_ map, so an exposure queued before setUnit may already carry the late unit. The guard avoids adding a second, contradictory record - it is not protecting a pristine one. Without that distinction a later reader can read the guard as an oversight and remove it. --- core-api/src/main/java/com/absmartly/sdk/Context.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 68f39a4..eb6dfa5 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -316,6 +316,11 @@ public void setUnit(@Nonnull final String unitType, @Nonnull final String uid) { // // Only unexposed assignments are evicted. Eviction creates a new exposed flag, so evicting an // already-exposed assignment could publish a duplicate or contradictory experiment exposure. + // This is not protecting a pristine record: publish() reads event.units from the live units_ + // map, so an exposure queued before this setUnit call may already carry the late unit. The + // decision behind it was still made without that unit, and recomputation cannot repair a + // record already queued - it can only add a second, conflicting one. The guard avoids + // compounding a degraded record, rather than pretending it is correct. private void invalidateAssignmentsPinnedWithMissingUnit(final String unitType) { final Iterator it = assignmentCache_.values().iterator(); while (it.hasNext()) { From 5d0650e3885fb450a1d3992bdae5c572e2a00cbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Tue, 1 Sep 2026 14:54:36 +0000 Subject: [PATCH 42/49] fix: settle exposure state across late-unit retries --- .../main/java/com/absmartly/sdk/Context.java | 82 +++++-- .../com/absmartly/sdk/ContextHoldoutTest.java | 225 +++++++++++++++++- 2 files changed, 282 insertions(+), 25 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index eb6dfa5..f51d227 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -26,6 +26,11 @@ import com.absmartly.sdk.json.*; public class Context implements Closeable { + private static final int ASSIGNMENT_UNEXPOSED = 0; + private static final int ASSIGNMENT_EXPOSED = 1; + private static final int ASSIGNMENT_RETIRED = 2; + private static final int MAX_EXPOSURE_ATTEMPTS = 3; + public static Context create(@Nonnull final Clock clock, @Nonnull final ContextConfig config, @Nonnull final ScheduledExecutorService scheduler, @Nonnull final CompletableFuture dataFuture, @Nonnull final ContextDataProvider dataProvider, @@ -314,7 +319,7 @@ public void setUnit(@Nonnull final String unitType, @Nonnull final String uid) { // Holdouts in this snapshot are resolved using assignment.unitType, not each holdout's declared // unitType, so only that unit installation invalidates a null entry. // - // Only unexposed assignments are evicted. Eviction creates a new exposed flag, so evicting an + // Only unexposed assignments are evicted. Eviction creates a new exposure state, so evicting an // already-exposed assignment could publish a duplicate or contradictory experiment exposure. // This is not protecting a pristine record: publish() reads event.units from the live units_ // map, so an exposure queued before this setUnit call may already carry the late unit. The @@ -325,15 +330,13 @@ private void invalidateAssignmentsPinnedWithMissingUnit(final String unitType) { final Iterator it = assignmentCache_.values().iterator(); while (it.hasNext()) { final Assignment assignment = it.next(); - if (assignment.exposed.get()) { - continue; - } - final Assignment[] holdoutAssignments = assignment.holdoutAssignments; if ((holdoutAssignments != null) && unitType.equals(assignment.unitType)) { for (final Assignment holdoutAssignment : holdoutAssignments) { if (holdoutAssignment == null) { - it.remove(); + if (assignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, ASSIGNMENT_RETIRED)) { + it.remove(); + } break; } } @@ -407,12 +410,20 @@ public void setAttributes(@Nonnull final Map attributes) { public int getTreatment(@Nonnull final String experimentName) { checkReady(true); - final Assignment assignment = getAssignment(experimentName); - if (!assignment.exposed.get()) { - triggerExposure(assignment); - } + return getExposedTreatmentVariant(getAssignment(experimentName)); + } - return assignment.variant; + private int getExposedTreatmentVariant(final Assignment assignment) { + return exposeTreatmentAssignment(assignment).variant; + } + + private Assignment exposeTreatmentAssignment(final Assignment assignment) { + return settleExposure(assignment, new Function() { + @Override + public Assignment apply(final Assignment retired) { + return getAssignment(retired.name); + } + }); } // A suppressed assignment (held out by an applicable holdout) never publishes its own @@ -420,12 +431,12 @@ public int getTreatment(@Nonnull final String experimentName) { // either exposure still triggers evaluation of every holdout applicable to this unit type, // keeping both holdout arms symmetric regardless of which covered experiment triggered it. // The trigger loop below must run even if the exposure above throws (e.g. a - // ContextEventLogger implementation that throws): `exposed` is already CAS'd true by the + // ContextEventLogger implementation that throws): the exposure state is already CAS'd by the // time we get here, so a skipped holdout trigger would never be retried for this context's // life. Failures are collected and re-thrown once every holdout has had a chance to fire, // rather than swallowed or allowed to abort the loop early. - private void triggerExposure(final Assignment assignment) { - if (assignment.exposed.compareAndSet(false, true)) { + private int triggerExposure(final Assignment assignment) { + if (assignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, ASSIGNMENT_EXPOSED)) { RuntimeException failure = null; try { if (!assignment.suppressed) { @@ -443,7 +454,9 @@ private void triggerExposure(final Assignment assignment) { if (failure != null) { throw failure; } + return ASSIGNMENT_EXPOSED; } + return assignment.exposureState.get(); } // Fires every holdout applicable to the given assignment, independent of whether the covered @@ -483,7 +496,8 @@ private void triggerHoldoutExposure(final Experiment holdoutExperiment, final St } private void triggerHoldoutExposure(final Assignment holdoutAssignment) { - if ((holdoutAssignment != null) && holdoutAssignment.exposed.compareAndSet(false, true)) { + if ((holdoutAssignment != null) && holdoutAssignment.exposureState.compareAndSet(ASSIGNMENT_UNEXPOSED, + ASSIGNMENT_EXPOSED)) { enqueueExposure(holdoutAssignment); } } @@ -558,13 +572,9 @@ public Map> getVariableKeys() { public Object getVariableValue(@Nonnull final String key, final Object defaultValue) { checkReady(true); - final Assignment assignment = getVariableAssignment(key, false); + final Assignment assignment = exposeVariableAssignment(key, getVariableAssignment(key, false)); if (assignment != null) { if (assignment.variables != null) { - if (!assignment.exposed.get()) { - triggerExposure(assignment); - } - if (assignment.variables.containsKey(key)) { return assignment.variables.get(key); } @@ -573,6 +583,30 @@ public Object getVariableValue(@Nonnull final String key, final Object defaultVa return defaultValue; } + private Assignment exposeVariableAssignment(final String key, final Assignment assignment) { + return settleExposure(assignment, new Function() { + @Override + public Assignment apply(final Assignment retired) { + return getVariableAssignment(key, false); + } + }); + } + + private Assignment settleExposure(final Assignment assignment, final Function resolver) { + Assignment current = assignment; + for (int attempt = 0; current != null; ++attempt) { + if (triggerExposure(current) != ASSIGNMENT_RETIRED) { + return current; + } + if (attempt + 1 >= MAX_EXPOSURE_ATTEMPTS) { + // Exhaustion returns the attempted assignment that lost to a concurrent retirement. + return current; + } + current = resolver.apply(current); + } + return current; + } + public Object peekVariableValue(@Nonnull final String key, final Object defaultValue) { checkReady(true); @@ -888,14 +922,14 @@ private static class Assignment { Assignment[] holdoutAssignments; Map variables = Collections.emptyMap(); - final AtomicBoolean exposed = new AtomicBoolean(false); + final AtomicInteger exposureState = new AtomicInteger(ASSIGNMENT_UNEXPOSED); } // Pins the (iteration, unitType) a holdout's Assignment was computed against. This is the same // granularity experimentMatches uses for ordinary experiments: seedHi/seedLo/split are // deliberately excluded so a live seed or percentage edit - which does not change who is // covered - never invalidates an already-exposed unit's arm. Only an iteration bump, a genuine - // re-randomization epoch, replaces the cached verdict (and its `exposed` flag), exactly as it + // re-randomization epoch, replaces the cached verdict (and its exposure state), exactly as it // does for ordinary experiments. armCount is deliberately excluded here too: it is pinned on // Assignment itself (see armCount below), bundled with the arm number it was computed // against, so the cached arm is always interpreted under its own arity rather than being @@ -1524,11 +1558,11 @@ private void logError(Throwable error) { // experiment's own set of applicable holdouts is still current (holdoutSetMatches), is its // (id, iteration) pair - never a full Experiment comparison. Rationale: the governing // invariant is that a unit must never appear in both arms of the same holdout within one - // context's lifetime, which the once-per-context `exposed` AtomicBoolean on each cached + // context's lifetime, which the once-per-context exposure state on each cached // Assignment enforces only as long as the entry it lives on is not needlessly replaced. // seedHi/seedLo/split/name/variants/applications/audience/customFieldValues can all change // without altering who is a member, so none of them may invalidate the entry - doing so - // would reset `exposed` and let an already-exposed unit be re-assigned into the other arm on + // would reset exposure and let an already-exposed unit be re-assigned into the other arm on // the very next refresh. Only an iteration bump is a genuine re-randomization epoch and // legitimately replaces the entry (and thus resets exposure), matching how experimentMatches // already treats iteration for ordinary experiments. diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index ea7889a..808d8fd 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -2,6 +2,7 @@ import static org.junit.jupiter.api.Assertions.assertArrayEquals; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.eq; @@ -11,10 +12,13 @@ import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; +import java.lang.reflect.Field; import java.lang.reflect.Method; +import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java8.util.concurrent.CompletableFuture; +import java8.util.function.Function; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -215,6 +219,49 @@ Exposure holdoutExposure(String unitType, int id, String name, int variant) { return new Exposure(id, name, unitType, variant, clock.millis(), true, true, false, false, false, false); } + Object getAssignment(Context context, String experimentName) throws Exception { + final Method method = Context.class.getDeclaredMethod("getAssignment", String.class); + method.setAccessible(true); + return method.invoke(context, experimentName); + } + + Object exposeTreatmentAssignment(Context context, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("exposeTreatmentAssignment", assignment.getClass()); + method.setAccessible(true); + return method.invoke(context, assignment); + } + + int getExposedTreatmentVariant(Context context, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("getExposedTreatmentVariant", assignment.getClass()); + method.setAccessible(true); + return (Integer) method.invoke(context, assignment); + } + + Object exposeVariableAssignment(Context context, String key, Object assignment) throws Exception { + final Method method = Context.class.getDeclaredMethod("exposeVariableAssignment", String.class, + assignment.getClass()); + method.setAccessible(true); + return method.invoke(context, key, assignment); + } + + void retireAssignment(Object assignment) throws Exception { + final Field field = assignment.getClass().getDeclaredField("exposureState"); + field.setAccessible(true); + ((AtomicInteger) field.get(assignment)).set(2); + } + + int assignmentVariant(Object assignment) throws Exception { + final Field field = assignment.getClass().getDeclaredField("variant"); + field.setAccessible(true); + return field.getInt(assignment); + } + + Object settleExposure(Context context, Object assignment, Function resolver) throws Exception { + final Method method = Context.class.getDeclaredMethod("settleExposure", assignment.getClass(), Function.class); + method.setAccessible(true); + return method.invoke(context, assignment, resolver); + } + // A held-out unit gets control values and emits only the holdout exposure. @Test void heldOutUnitGetsControlValuesAndEmitsNoExposureForCoveredExperiment() { @@ -1125,7 +1172,7 @@ void setUnitInvalidatesNullHoldoutSnapshotSoTheHoldoutExposureIsNotLostForever() // Assignment, the next getTreatment builds a fresh one with exposed==false, and the // experiment's exposure is published twice for the same unit. @Test - void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() { + void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() throws Exception { final String coveredUnitType = "user_id"; final Experiment experiment = coveredBy( newExperiment(1, "exp_exposed_before_unit", coveredUnitType, 2), 11); @@ -1137,11 +1184,13 @@ void setUnitDoesNotEvictAlreadyExposedAssignmentSoNoDuplicateExposure() { assertEquals(2, context.getTreatment("exp_exposed_before_unit")); assertEquals(1, context.getPendingCount()); + final Object exposed = getAssignment(context, "exp_exposed_before_unit"); context.setUnit(coveredUnitType, UID); assertEquals(2, context.getTreatment("exp_exposed_before_unit")); assertEquals(1, context.getPendingCount()); // unchanged: no duplicate exposure + assertSame(exposed, getAssignment(context, "exp_exposed_before_unit")); } // Selectivity: an unrelated unit type must never touch cache entries at all, exposed or not. @@ -1251,6 +1300,158 @@ void setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses() { verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } + @Test + void detachedTreatmentAssignmentCannotPublishBesideSuppressingReplacement() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_suppressed"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_suppressed", unitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + assertEquals(0, context.getTreatment(experimentName)); + assertEquals(1, context.getPendingCount()); + + exposeTreatmentAssignment(context, retained); + assertEquals(1, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, holdoutExposure(unitType, 11, "holdout_detached_suppressed", 0))); + } + + @Test + void detachedTreatmentAssignmentRetriesCurrentEntryWithoutLosingExposure() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_retry"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_retry", unitType, + HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + exposeTreatmentAssignment(context, retained); + + assertEquals(2, context.getPendingCount()); + assertEquals(2, context.getTreatment(experimentName)); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, + new Exposure(1, experimentName, unitType, 2, clock.millis(), true, true, false, true, false, + false), + holdoutExposure(unitType, 11, "holdout_detached_retry", 1))); + } + + @Test + void detachedTreatmentReturnsTheReplacementVariant() throws Exception { + final String unitType = "user_id"; + final String experimentName = "exp_detached_variant"; + final Experiment experiment = coveredBy(newExperiment(1, experimentName, unitType, 2), 11); + final Experiment holdout = newHoldout(11, "holdout_detached_variant", unitType, + HOLDOUT_A_SEED_HI, HOLDOUT_A_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, experiment)); + + final Object retained = getAssignment(context, experimentName); + context.setUnit(unitType, UID); + + assertEquals(0, getExposedTreatmentVariant(context, retained)); + assertEquals(1, context.getPendingCount()); + } + + @Test + void treatmentExposureSettlesAfterTwoConsecutiveRetirements() throws Exception { + final Experiment first = newExperiment(1, "exp_first", 2); + final Experiment second = newExperiment(2, "exp_second", 1); + final Experiment settled = newExperiment(3, "exp_settled", 1); + final Context context = createReadyContext(contextDataOf(first, second, settled)); + final Object firstAssignment = getAssignment(context, first.name); + final Object secondAssignment = getAssignment(context, second.name); + final Object settledAssignment = getAssignment(context, settled.name); + retireAssignment(firstAssignment); + retireAssignment(secondAssignment); + + final Object result = settleExposure(context, firstAssignment, new Function() { + int call; + + @Override + public Object apply(Object ignored) { + return call++ == 0 ? secondAssignment : settledAssignment; + } + }); + + assertSame(settledAssignment, result); + assertEquals(1, assignmentVariant(result)); + assertEquals(1, context.getPendingCount()); + } + + @Test + void exposureSettlementStopsAfterTheBoundedNumberOfRetirements() throws Exception { + final Experiment experiment = newExperiment(1, "exp_bounded_retry", 1); + final Context context = createReadyContext(contextDataOf(experiment)); + final Object retired = getAssignment(context, experiment.name); + retireAssignment(retired); + final AtomicInteger resolutions = new AtomicInteger(); + + settleExposure(context, retired, new Function() { + @Override + public Object apply(Object ignored) { + if (resolutions.incrementAndGet() > 2) { + throw new AssertionError("exposure retry exceeded its bound"); + } + return retired; + } + }); + + assertEquals(2, resolutions.get()); + assertEquals(0, context.getPendingCount()); + } + + @Test + void detachedVariableAssignmentReresolvesTheKeyToANewWinningExperiment() throws Exception { + final String unitType = "user_id"; + final Experiment original = coveredBy(newExperiment(2, "exp_detached_variable_old", unitType, 1), 11); + original.variants[1].config = "{\"detached_var\":\"old\"}"; + final Experiment holdout = newHoldout(11, "holdout_detached_variable", unitType, + HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, "full"); + final Context context = createReadyContextWithoutUnits( + contextDataOf(new Experiment[]{holdout}, original)); + + final Object retained = getAssignment(context, original.name); + context.setUnit(unitType, UID); + + final Experiment winner = coveredBy(newExperiment(1, "exp_detached_variable_winner", unitType, 1), 11); + winner.variants[1].config = "{\"detached_var\":\"new winner\"}"; + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, winner, original)); + refreshing.join(); + + final Object resolved = exposeVariableAssignment(context, "detached_var", retained); + assertEquals(1, assignmentVariant(resolved)); + assertEquals(2, context.getPendingCount()); + assertEquals("new winner", context.getVariableValue("detached_var", "default")); + assertEquals(2, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(unitType, UID, + holdoutExposure(unitType, 11, "holdout_detached_variable", 1), + new Exposure(1, winner.name, unitType, 1, clock.millis(), true, true, false, true, false, + false))); + } + // Kills a narrower variant of the "re-resolve the null live at trigger time" mutant that // setUnitRecomputesSuppressionCoherentlyWhenLateResolvedHoldoutSuppresses cannot reach: once // an Assignment is already exposed, setUnit's `!exposed` guard deliberately leaves its null @@ -1726,6 +1927,28 @@ void throwingLoggerDoesNotPermanentlyLoseSiblingHoldoutExposure() { assertEquals(2, context.getPendingCount()); } + @Test + void throwingLoggerDoesNotMakeExperimentOrHoldoutExposureRetryable() { + doThrow(new RuntimeException("boom")).when(eventLogger).handleEvent(any(), any(), + org.mockito.ArgumentMatchers.argThat(o -> (o instanceof Exposure) && (((Exposure) o).id == 1))); + + final Experiment experiment = coveredBy(newExperiment(1, "exp_throwing_logger_terminal"), 11); + final Experiment holdout = newHoldout(11, "holdout_terminal", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); + final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); + + assertThrows(RuntimeException.class, () -> context.getTreatment(experiment.name)); + Mockito.reset(eventLogger); + assertEquals(NORMAL_VARIANT, context.getTreatment(experiment.name)); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, + publishedEvent(UID, + new Exposure(1, experiment.name, UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), + holdoutExposure(11, "holdout_terminal", 1))); + } + // Regression test: a throwing logger for a suppressed experiment's ONLY holdout exposure must // still schedule a flush. There is no ordinary exposure to fall back on for scheduling (the // covered experiment is suppressed), so enqueueExposure's own setTimeout() call is the only From 7d9ea907891c0e762ec4e8c45c3c877f7df892ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Tue, 1 Sep 2026 18:04:37 +0000 Subject: [PATCH 43/49] test: replace stale audience cache signal Main's audience-aware cache validity now correctly invalidates the attribute-change signal. Use a same-iteration split refresh so the redundant setUnit test still distinguishes retained and evicted assignments. --- .../com/absmartly/sdk/ContextHoldoutTest.java | 37 +++++++++---------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index 808d8fd..e560c6f 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -1558,11 +1558,10 @@ void setUnitCalledTwiceWithSameUidIsANoOpAndDoesNotThrow() { // Kills the "evict on holdoutAssignments != null alone" mutant (dropping the check that some // entry is actually null): a cached, unexposed assignment whose holdout snapshot is fully // resolved (no null entries, because the unit was already present when it was computed) must - // survive a redundant setUnit call for that same unit type. The distinguishing signal is an - // attribute change made between the peek and the redundant setUnit call: experimentMatches - // never considers attributes, so a retained cache entry keeps the audience verdict (and thus - // the variant) it was computed with, while a wrongly-evicted entry would recompute against - // the now-matching attribute and land on a different variant entirely. + // survive a redundant setUnit call for that same unit type. The distinguishing signal is a + // same-iteration split refresh, which both experimentMatches and audienceMatches ignore: a + // retained entry stays pinned to its original variant, while a wrongly-evicted entry recomputes + // against the new split and lands on a different variant. // // Fails against `(holdoutAssignments != null) && unitType.equals(assignment.unitType)` alone: // the array is non-null (one resolved holdout entry) and the unit type matches, so the entry @@ -1570,37 +1569,35 @@ void setUnitCalledTwiceWithSameUidIsANoOpAndDoesNotThrow() { // re-decides the experiment. @Test void setUnitRedundantCallDoesNotEvictFullyResolvedUnexposedHoldoutSnapshot() { - final String audience = "{\"filter\":[{\"gte\":[{\"var\":\"age\"},{\"value\":20}]}]}"; - final Experiment experiment = coveredBy(newExperiment(1, "exp_redundant_set_unit"), 11); - experiment.audienceStrict = true; - experiment.audience = audience; final Experiment holdout = newHoldout(11, "holdout_a", HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO); // not held out final Context context = createReadyContext(contextDataOf(new Experiment[]{holdout}, experiment)); - context.setAttribute("age", 5); // mismatches: forces control regardless of holdout // resolved with the unit already present: holdoutAssignments holds no null entry. - assertEquals(0, context.peekTreatment("exp_redundant_set_unit")); // audience mismatch -> control + assertEquals(NORMAL_VARIANT, context.peekTreatment("exp_redundant_set_unit")); assertEquals(0, context.getPendingCount()); // peek never exposes - context.setAttribute("age", 25); // now matches; experimentMatches ignores attributes though + final Experiment refreshedExperiment = coveredBy(newExperiment(1, "exp_redundant_set_unit"), 11); + refreshedExperiment.split = new double[]{1.0, 0.0}; // a recomputation must land in variant 0 + final CompletableFuture refreshFuture = new CompletableFuture<>(); + when(dataProvider.getContextData()).thenReturn(refreshFuture); + final CompletableFuture refreshing = context.refreshAsync(); + refreshFuture.complete(contextDataOf(new Experiment[]{holdout}, refreshedExperiment)); + refreshing.join(); + context.setUnit(UNIT_TYPE, UID); // redundant: same unit type, same uid already installed - // still the pinned, audience-mismatched decision: a wrongly-evicted entry would recompute - // under the now-matching attribute and land on NORMAL_VARIANT instead. - assertEquals(0, context.getTreatment("exp_redundant_set_unit")); - assertEquals(2, context.getPendingCount()); // own exposure (audience-mismatch) + holdout's + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_redundant_set_unit")); + assertEquals(2, context.getPendingCount()); // own exposure + holdout's when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); context.publish(); final PublishEvent expected = publishedEvent(UID, - new Exposure(1, "exp_redundant_set_unit", UNIT_TYPE, 0, clock.millis(), false, true, false, false, - false, true), + new Exposure(1, "exp_redundant_set_unit", UNIT_TYPE, NORMAL_VARIANT, clock.millis(), true, true, false, + false, false, false), holdoutExposure(11, "holdout_a", 1)); - expected.attributes = new Attribute[]{new Attribute("age", 5, clock.millis()), - new Attribute("age", 25, clock.millis())}; verify(eventHandler, Mockito.timeout(5000).times(1)).publish(context, expected); } From ac76466ec4483e3f43d39ea05a41ea1a0d69450f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Tue, 1 Sep 2026 23:22:10 +0000 Subject: [PATCH 44/49] style: fix import order flagged by spotless --- .../src/test/java/com/absmartly/sdk/ContextHoldoutTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index e560c6f..ff9e165 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -14,9 +14,9 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; -import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; import java8.util.concurrent.CompletableFuture; import java8.util.function.Function; From b05cb26f97bbade757459f0c371fa72dbf7794ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Wed, 2 Sep 2026 02:19:36 +0000 Subject: [PATCH 45/49] fix: narrow custom-assignment cache invalidation to the pinned custom value variantForcedRegardlessOfCustom() (introduced in 5ec87d4) bypassed cache invalidation entirely for full-on, ineligible, and unassigned entries once ANY custom assignment was on file, including one set after the cached entry was resolved. A legitimate custom-assignment change on a full-on entry was then silently swallowed: getAssignment kept returning the stale cached Assignment forever, so getTreatment never re-exposed (conformance scenario 66 - Custom Assignment - Clear Cache). Replace the broad bypass with a pinned comparison: Assignment now records the cassignments_ entry (customAssignment) that was live when it was resolved, and the fast path compares the live entry against that pinned value instead of against the resolved variant. This keeps forced entries (where the custom value can never equal the resolved variant) cache-valid across repeated calls with the same custom assignment, while still invalidating exactly when the custom assignment itself changes. Add setCustomAssignmentAfterExposureReExposesFullOnAssignment, which fails under the reverted broad condition (pendingCount stays at 1 instead of reaching 2) and passes with the narrowed one. --- .../main/java/com/absmartly/sdk/Context.java | 22 ++++++++-------- .../java/com/absmartly/sdk/ContextTest.java | 25 +++++++++++++++++++ 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 3754221..007d797 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1003,16 +1003,6 @@ private static boolean holdoutSetMatches(final Experiment[] a, final Experiment[ return true; } - // A custom assignment can only take effect on the normal, traffic-eligible assignment path. - // When the cached variant was forced by a higher-precedence rule — a full-on variant, traffic - // ineligibility, a strict audience mismatch, or holdout suppression (which clears `assigned`) — - // the custom value can never equal that variant, so comparing the two would spuriously - // invalidate the cache and re-expose on every getTreatment call. Treat those forced - // assignments as cache-valid regardless of the custom assignment. - private static boolean variantForcedRegardlessOfCustom(final Assignment assignment) { - return assignment.fullOn || !assignment.eligible || !assignment.assigned; - } - private boolean audienceMatches(final Experiment experiment, final Assignment assignment) { if (experiment.audience != null && experiment.audience.length() > 0) { if (attrsSeq_.get() > assignment.attrsSeq) { @@ -1045,6 +1035,14 @@ private static class Assignment { boolean eligible; boolean fullOn; boolean custom; + // The cassignments_ entry read while this Assignment was resolved (null if none was set + // yet), regardless of whether the resolution path actually consulted it - a full-on or + // traffic-ineligible variant never does. Comparing the live entry against this pinned + // value, rather than against the resolved variant, is what tells the cache apart from a + // custom assignment that legitimately changes the outcome: the live value can equal the + // resolved variant by construction on the eligible path, but a forced variant can never + // equal a newly-set custom value, which would otherwise invalidate on every call. + Integer customAssignment; boolean audienceMismatch; // Held out by a union of applicable holdouts: no exposure for this experiment, control @@ -1121,8 +1119,7 @@ && holdoutSetMatches((experiment != null) ? experiment.holdouts : null, // previously not-running experiment return assignment; } - } else if ((custom == null) || variantForcedRegardlessOfCustom(assignment) - || custom == assignment.variant) { + } else if (Objects.equals(custom, assignment.customAssignment)) { if (experimentMatches(experiment, assignment) && audienceMatches(experiment.data, assignment)) { // assignment up-to-date @@ -1146,6 +1143,7 @@ && audienceMatches(experiment.data, assignment)) { final Assignment assignment = new Assignment(); assignment.name = experimentName; assignment.eligible = true; + assignment.customAssignment = custom; if (override != null) { if (experiment != null) { diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java index 311540c..a37287c 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextTest.java @@ -846,6 +846,31 @@ void setCustomAssignmentClearsAssignmentCache() { assertEquals(cassignments.size() * 3, context.getPendingCount()); } + // Regression test for the java-sdk#12 cache bug: a full-on variant can never equal the custom + // value on file, so the cache-validity check must not compare them directly (that would + // invalidate on every call once any custom assignment is set - see + // setCustomAssignmentDoesNotOverrideFullOnOrNotEligibleAssignments's sibling history) nor + // ignore the custom assignment outright (that would mask a legitimate change and never + // re-expose, which is the regression: exposure count must go 1 -> 2 -> 2, not 1 -> 1 -> 1). + @Test + void setCustomAssignmentAfterExposureReExposesFullOnAssignment() { + final Context context = createReadyContext(); + + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(1, context.getPendingCount()); + + // the custom value can never win against a full-on variant, but setting one is still a + // change the cache must react to: it owes exactly one fresh exposure for it + context.setCustomAssignment("exp_test_fullon", 0); + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(2, context.getPendingCount()); + + // repeating the same custom assignment must not re-invalidate the cache + context.setCustomAssignment("exp_test_fullon", 0); + assertEquals(2, context.getTreatment("exp_test_fullon")); + assertEquals(2, context.getPendingCount()); + } + @Test void setCustomAssignmentsBeforeReady() { final Context context = createContext(dataFuture); From 1b2501c66e667e2a98063812c2472f17b26ac7b9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Wed, 2 Sep 2026 02:44:47 +0000 Subject: [PATCH 46/49] fix: key holdout assignment cache by (id, effective unit type) holdoutAssignmentCache_ was keyed by holdout id alone, but getHoldoutAssignment resolves each holdout against the covered experiment's effective unit type, which need not equal the holdout's own declared unitType. When one holdout id covers experiments with two different unit types, alternating lookups (A1 -> B -> A2) evicted each other's cache slot on every miss: B's lookup replaced A's entry because matches() failed only on unitType, discarding A's exposureState. The next A lookup then missed, built a brand-new Assignment with a fresh exposureState, and republished that holdout's own exposure a second time for the same unit. Introduce HoldoutCacheKey, a composite (id, unitType) key, so each (holdout, effective unit) pair gets its own cache slot and its own once-per-context exposure state. unitType is dropped from HoldoutAssignment itself since the key now carries it exclusively; matches() is reduced to the iteration check it still owns. The double-checked locking (read under readLock, recheck under writeLock) and the id-resolution-before-matches() behavior in getHoldoutAssignment are unchanged. Regression test reproduces the exact A1 -> B -> A2 alternation Pedro found: one holdout id/iteration covering two unit types, 3 covered experiments. Confirms 5 pending events (not 6) and the holdout's own exposure published once per unit type (2 total), not once per alternation (3). --- .../main/java/com/absmartly/sdk/Context.java | 89 +++++++++++++------ .../com/absmartly/sdk/ContextHoldoutTest.java | 62 +++++++++++++ 2 files changed, 126 insertions(+), 25 deletions(-) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index 007d797..d9c1527 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -1065,28 +1065,63 @@ private static class Assignment { final AtomicInteger exposureState = new AtomicInteger(ASSIGNMENT_UNEXPOSED); } - // Pins the (iteration, unitType) a holdout's Assignment was computed against. This is the same - // granularity experimentMatches uses for ordinary experiments: seedHi/seedLo/split are - // deliberately excluded so a live seed or percentage edit - which does not change who is - // covered - never invalidates an already-exposed unit's arm. Only an iteration bump, a genuine - // re-randomization epoch, replaces the cached verdict (and its exposure state), exactly as it - // does for ordinary experiments. armCount is deliberately excluded here too: it is pinned on - // Assignment itself (see armCount below), bundled with the arm number it was computed - // against, so the cached arm is always interpreted under its own arity rather than being - // invalidated and re-exposed under a new one for a same-iteration arity change. + // A holdout's own Assignment is cached per (id, effective unit type): one holdout id can cover + // experiments with different unit types (getHoldoutAssignment receives the covered + // experiment's unitType, not the holdout's own declared unitType - see the comment on + // getHoldoutAssignment), and each such unit type is a distinct suppression decision with its + // own once-per-context exposure state. Keying by id alone let two unit types evict each + // other's slot on every alternating lookup, discarding the loser's exposureState and + // re-publishing that holdout's membership on the next recomputation. The unit type is part of + // this key rather than a field compared inside matches(): duplicating it as both would leave + // two sources of truth for the same identity. + private static final class HoldoutCacheKey { + final int id; + final String unitType; + + HoldoutCacheKey(int id, String unitType) { + this.id = id; + this.unitType = unitType; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (!(o instanceof HoldoutCacheKey)) { + return false; + } + final HoldoutCacheKey other = (HoldoutCacheKey) o; + return (id == other.id) && unitType.equals(other.unitType); + } + + @Override + public int hashCode() { + return (31 * id) + unitType.hashCode(); + } + } + + // Pins the iteration a holdout's Assignment was computed against, for the unit type already + // fixed by this entry's HoldoutCacheKey. This is the same granularity experimentMatches uses + // for ordinary experiments: seedHi/seedLo/split are deliberately excluded so a live seed or + // percentage edit - which does not change who is covered - never invalidates an already- + // exposed unit's arm. Only an iteration bump, a genuine re-randomization epoch, replaces the + // cached verdict (and its exposure state), exactly as it does for ordinary experiments. + // armCount is deliberately excluded here too: it is pinned on Assignment itself (see armCount + // below), bundled with the arm number it was computed against, so the cached arm is always + // interpreted under its own arity rather than being invalidated and re-exposed under a new + // one for a same-iteration arity change. private static class HoldoutAssignment { final Assignment assignment; final int iteration; - final String unitType; - HoldoutAssignment(Assignment assignment, Experiment holdout, String unitType) { + HoldoutAssignment(Assignment assignment, Experiment holdout) { this.assignment = assignment; this.iteration = holdout.iteration; - this.unitType = unitType; } - boolean matches(Experiment current, String currentUnitType) { - return (iteration == current.iteration) && unitType.equals(currentUnitType); + boolean matches(Experiment current) { + return iteration == current.iteration; } } @@ -1357,11 +1392,12 @@ private Experiment getHoldoutById(final int holdoutId) { } } - // The holdout's own assignment is cached by holdout id rather than name, since holdout - // entries live outside the experiments index and are shared by reference across every - // covered experiment. Only an iteration change invalidates the cache entry (see - // HoldoutAssignment), matching experimentMatches's treatment of ordinary experiments and - // guaranteeing an already-exposed unit's arm survives any seed, split or cosmetic edit. + // The holdout's own assignment is cached by (id, effective unit type) rather than name, since + // holdout entries live outside the experiments index and are shared by reference across every + // covered experiment - see HoldoutCacheKey for why the unit type is part of the identity. + // Only an iteration change invalidates the cache entry (see HoldoutAssignment), matching + // experimentMatches's treatment of ordinary experiments and guaranteeing an already-exposed + // unit's arm survives any seed, split or cosmetic edit. // // The `holdout` parameter can be a stale Experiment reference: callers reach this method via // a cached Assignment.holdouts array that may predate the most recent setData (e.g. the @@ -1383,8 +1419,8 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u } final Experiment liveHoldout = resolveLiveHoldout(holdout); - final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); - if ((cached != null) && cached.matches(liveHoldout, unitType)) { + final HoldoutAssignment cached = holdoutAssignmentCache_.get(new HoldoutCacheKey(holdout.id, unitType)); + if ((cached != null) && cached.matches(liveHoldout)) { return cached.assignment; } } finally { @@ -1403,8 +1439,8 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u } final Experiment liveHoldout = resolveLiveHoldout(holdout); - final HoldoutAssignment cached = holdoutAssignmentCache_.get(holdout.id); - if ((cached != null) && cached.matches(liveHoldout, unitType)) { + final HoldoutAssignment cached = holdoutAssignmentCache_.get(new HoldoutCacheKey(holdout.id, unitType)); + if ((cached != null) && cached.matches(liveHoldout)) { return cached.assignment; } @@ -1421,7 +1457,8 @@ private Assignment getHoldoutAssignment(final Experiment holdout, final String u assignment.variant = assigner.assign(liveHoldout.split, liveHoldout.seedHi, liveHoldout.seedLo); assignment.armCount = (liveHoldout.split != null) ? liveHoldout.split.length : 0; - holdoutAssignmentCache_.put(liveHoldout.id, new HoldoutAssignment(assignment, liveHoldout, unitType)); + holdoutAssignmentCache_.put(new HoldoutCacheKey(liveHoldout.id, unitType), + new HoldoutAssignment(assignment, liveHoldout)); return assignment; } finally { @@ -1754,7 +1791,9 @@ private Map buildAttributesMap() { // the very next refresh. Only an iteration bump is a genuine re-randomization epoch and // legitimately replaces the entry (and thus resets exposure), matching how experimentMatches // already treats iteration for ordinary experiments. - private final Map holdoutAssignmentCache_ = new HashMap(); + // + // Keyed by (id, effective unit type) rather than id alone: see HoldoutCacheKey. + private final Map holdoutAssignmentCache_ = new HashMap(); private final ReentrantLock eventLock_ = new ReentrantLock(); private final ArrayList exposures_ = new ArrayList(); diff --git a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java index ff9e165..77b05f3 100644 --- a/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/ContextHoldoutTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.Mockito; import com.absmartly.sdk.internal.hashing.Hashing; @@ -1521,6 +1522,67 @@ void setUnitEvictsAssignmentWithMultipleHoldoutsDeclaringDifferentUnitTypes() { assertEquals(4, context.getPendingCount()); // experiment + 3 holdout exposures } + // One holdout id covering experiments with two different effective unit types must keep an + // independent Assignment (and exposure state) per unit type: alternating lookups between the + // two units must never evict each other's slot. Before the fix, getHoldoutAssignment cached + // solely by holdout id, so the sequence A1 -> B -> A2 (a first experiment resolving unit type + // A, then one resolving unit type B, then a second experiment resolving unit type A again) + // evicted A's slot when B was resolved, forcing A2 to recompute against a brand-new + // Assignment object with a fresh, unexposed exposureState - and republish the holdout's own + // exposure for unit type A a second time. This exactly reproduces the reported A1 -> B -> A2 + // alternation: one holdout id/iteration, two unit types, three covered-experiment + // resolutions in that order. + @Test + void oneHoldoutCoveringTwoUnitTypesKeepsIndependentExposureStatePerUnitType() { + final String unitTypeA = "user_id"; + final String unitTypeB = "session_id"; + final Experiment expA1 = coveredBy(newExperiment(1, "exp_unit_a1", unitTypeA, 0), 11); + final Experiment expB = coveredBy(newExperiment(2, "exp_unit_b", unitTypeB, 0), 11); + final Experiment expA2 = coveredBy(newExperiment(3, "exp_unit_a2", unitTypeA, 0), 11); + // not held out for either unit under HOLDOUT_B seeds, isolating the alternation bug from + // suppression: every covered experiment assigns normally, only the holdout's own + // once-per-(id, unit type) exposure is under test. + final Experiment holdout = newHoldout(11, "holdout_shared", unitTypeA, HOLDOUT_B_SEED_HI, HOLDOUT_B_SEED_LO, + "full"); + + final ContextConfig config = ContextConfig.create().setUnit(unitTypeA, UID).setUnit(unitTypeB, UID); + final Context context = createReadyContext(config, + contextDataOf(new Experiment[]{holdout}, expA1, expB, expA2)); + + // A1: resolves and caches holdout_shared under (11, unitTypeA); fires its own exposure and + // the holdout's. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_a1")); + // B: resolves (11, unitTypeB). Pre-fix, this evicted the (11, unitTypeA) slot. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_b")); + // A2: a DIFFERENT experiment also covered by holdout 11 under unitTypeA. Pre-fix, the + // evicted slot forces a fresh, unexposed Assignment object here, so triggering exp_unit_a2 + // re-fires holdout_shared's exposure a second time for the same unit type. + assertEquals(NORMAL_VARIANT, context.getTreatment("exp_unit_a2")); + + // Pedro's reproduction: 3 covered-experiment exposures + 1 holdout exposure per unit type + // = 5 pending events, not 6. + assertEquals(5, context.getPendingCount()); + + when(eventHandler.publish(any(), any())).thenReturn(CompletableFuture.completedFuture(null)); + context.publish(); + + final ArgumentCaptor captor = ArgumentCaptor.forClass(PublishEvent.class); + verify(eventHandler, Mockito.timeout(5000).times(1)).publish(Mockito.eq(context), captor.capture()); + + final Exposure[] exposures = captor.getValue().exposures; + assertEquals(5, exposures.length); + + // holdout_shared's own exposure must be published exactly once per unit type (twice + // total), never once per alternation (which would be three times: A1, B, A2). + int holdoutExposureCount = 0; + for (final Exposure exposure : exposures) { + if (exposure.id == 11) { + ++holdoutExposureCount; + } + } + assertEquals(2, holdoutExposureCount); + } + // setUnit must be a no-op with respect to the assignment cache when nothing has been cached // yet for this context - no exception, and the subsequent assignment behaves normally. @Test From 6a64267b1c9a5fc86995d7be6055b8df3e9b207a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Wed, 2 Sep 2026 11:37:53 +0000 Subject: [PATCH 47/49] fix: use the bundled Objects shim to hold the Java 1.6 API floor Context.java has no explicit Objects import, so the wildcard java.util.* bound Objects.equals to the JDK class, which is Java 7+. Animal Sniffer enforces a Java 1.6 floor on core-api production source (gradle/compatibility.gradle), so the build failed with an undefined reference. Import com.absmartly.sdk.java.util.Objects instead, the same shim every json model class already uses. --- core-api/src/main/java/com/absmartly/sdk/Context.java | 1 + 1 file changed, 1 insertion(+) diff --git a/core-api/src/main/java/com/absmartly/sdk/Context.java b/core-api/src/main/java/com/absmartly/sdk/Context.java index d9c1527..389719b 100644 --- a/core-api/src/main/java/com/absmartly/sdk/Context.java +++ b/core-api/src/main/java/com/absmartly/sdk/Context.java @@ -28,6 +28,7 @@ import com.absmartly.sdk.internal.hashing.Hashing; import com.absmartly.sdk.java.nio.charset.StandardCharsets; import com.absmartly.sdk.java.time.Clock; +import com.absmartly.sdk.java.util.Objects; import com.absmartly.sdk.json.*; public class Context implements Closeable { From 013a800e6ba2773277cf0cea4dc7520fca0e70a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Wed, 2 Sep 2026 13:33:46 +0000 Subject: [PATCH 48/49] test: stop requiring holdoutType in the deserializer fixture abs#4939 removed holdoutType from the collector context response, but the holdouts fixture still carried it on both holdouts and deserializeHoldouts asserted both values - pinning a payload the server no longer produces. Dropped from holdout B only, deliberately. Holdout A keeps it so the test still pins that a legacy payload from an older collector deserializes cleanly during rollout; holdout B pins the new shape, where arity comes from split.length. --- .../absmartly/sdk/DefaultContextDataDeserializerTest.java | 5 ++++- core-api/src/test/resources/holdouts_context.json | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index 131494c..f4c40ee 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -187,7 +187,10 @@ void deserializeHoldouts() { }; holdoutB.audienceStrict = false; holdoutB.audience = null; - holdoutB.holdoutType = "all_full_on"; + // holdoutType is no longer served (abs#4939): the collector omits it and arity comes + // from split.length. Left unset here so the equality assertion pins that a holdout + // without the field deserializes cleanly. holdoutA still carries it, pinning that a + // legacy payload from an older collector is still tolerated. final ContextData expected = new ContextData( new Experiment[]{experiment}, diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json index 910778c..a398fdb 100644 --- a/core-api/src/test/resources/holdouts_context.json +++ b/core-api/src/test/resources/holdouts_context.json @@ -102,8 +102,7 @@ "config":null } ], - "audience": null, - "holdoutType":"all_full_on" + "audience": null } ] } From 944f700fc11bdd897030da221b089155d8b6b26f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?M=C3=A1rcio=20Martins?= Date: Wed, 2 Sep 2026 15:09:43 +0000 Subject: [PATCH 49/49] test: pin both the slim and legacy holdout wire shapes The collector is moving holdouts to a dedicated 7-field DTO (id, name, unitType, iteration, seedHi, seedLo, split), so the fixture no longer matched what the server sends: both holdouts still carried trafficSeed*/trafficSplit/fullOnVariant/ variants/audience. Slimmed holdout 12 to the new shape and left holdout 11 fat, so the deserializer test pins both directions - the new payload deserializes cleanly (absent primitive ints default to 0, arrays to null), and a legacy payload from an older collector is still tolerated during rollout. --- .../DefaultContextDataDeserializerTest.java | 8 ++----- .../src/test/resources/holdouts_context.json | 24 +------------------ 2 files changed, 3 insertions(+), 29 deletions(-) diff --git a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java index f4c40ee..fb3b2cf 100644 --- a/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java +++ b/core-api/src/test/java/com/absmartly/sdk/DefaultContextDataDeserializerTest.java @@ -178,13 +178,9 @@ void deserializeHoldouts() { holdoutB.seedHi = 1; holdoutB.seedLo = 222; holdoutB.split = new double[]{0.05, 0.05, 0.9}; - holdoutB.trafficSplit = new double[]{0.0, 1.0}; + holdoutB.trafficSplit = null; holdoutB.fullOnVariant = 0; - holdoutB.variants = new ExperimentVariant[]{ - new ExperimentVariant("A", null), - new ExperimentVariant("B", null), - new ExperimentVariant("C", null) - }; + holdoutB.variants = null; holdoutB.audienceStrict = false; holdoutB.audience = null; // holdoutType is no longer served (abs#4939): the collector omits it and arity comes diff --git a/core-api/src/test/resources/holdouts_context.json b/core-api/src/test/resources/holdouts_context.json index a398fdb..92f1dbf 100644 --- a/core-api/src/test/resources/holdouts_context.json +++ b/core-api/src/test/resources/holdouts_context.json @@ -80,29 +80,7 @@ 0.05, 0.05, 0.9 - ], - "trafficSeedHi":0, - "trafficSeedLo":0, - "trafficSplit":[ - 0.0, - 1.0 - ], - "fullOnVariant":0, - "variants":[ - { - "name":"A", - "config":null - }, - { - "name":"B", - "config":null - }, - { - "name":"C", - "config":null - } - ], - "audience": null + ] } ] }