From 5503404c76f8a96532bbb06fea7c22acf0d80a14 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Wed, 16 Sep 2026 19:01:41 -0400 Subject: [PATCH 1/5] Changes to support filtering/sorting for germplasm - Additional changes made to support sorting by numbers, especially for additionalInfo fields - Bugs fixed when left join fetches were made but filters were required for specific fields --- .../model/germ/GermplasmSearchRequest.java | 27 +++++++++++++++++++ .../BrAPITestServer/model/dto/EntityType.java | 4 ++- .../model/entity/germ/GermplasmEntity.java | 19 +++++++++++++ .../service/SearchQueryBuilder.java | 22 ++++++++------- .../service/germ/GermplasmService.java | 13 +++++---- 5 files changed, 70 insertions(+), 15 deletions(-) diff --git a/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java b/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java index 79bbf045..d1acd966 100644 --- a/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java +++ b/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java @@ -3,12 +3,34 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import io.swagger.model.SearchRequest; +import org.brapi.test.BrAPITestServer.model.dto.EntityColumnNameAndType; +import org.brapi.test.BrAPITestServer.model.dto.EntityType; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.Objects; public class GermplasmSearchRequest extends SearchRequest { + + // Key - allowed sort or field filter name for this entity + // Value = entity field name and type that represents the submitted field. Used later on in query building. + private static final Map ALLOWED_SORT_AND_FILTER_FIELDS = + Map.ofEntries( + // accessionNumber = GID for DeltaBreed + Map.entry("accessionNumber", new EntityColumnNameAndType("accessionNumber", EntityType.NUMBER)), + Map.entry("defaultDisplayName", new EntityColumnNameAndType("defaultDisplayName", EntityType.TEXT)), + Map.entry("breedingMethod", new EntityColumnNameAndType("methodName", EntityType.TEXT)), + Map.entry("seedSource", new EntityColumnNameAndType("seedSource", EntityType.TEXT)), + Map.entry("pedigree",new EntityColumnNameAndType("pedigree.pedigreeString", EntityType.TEXT)), + Map.entry("femaleParentGID", new EntityColumnNameAndType("femaleParentGid", EntityType.NUMBER)), + Map.entry("maleParentGID", new EntityColumnNameAndType("maleParentGid", EntityType.NUMBER)), + Map.entry("createdDate", new EntityColumnNameAndType("createdDate", EntityType.TEXT)), + Map.entry("createdBy", new EntityColumnNameAndType("createdBy", EntityType.TEXT)), + Map.entry("importEntryNumber", new EntityColumnNameAndType("importEntryNumber", EntityType.NUMBER)), + Map.entry("synonyms", new EntityColumnNameAndType("*synonyms.synonym", EntityType.TEXT)) + ); + @JsonProperty("accessionNumbers") private List accessionNumbers = null; @@ -621,4 +643,9 @@ public Integer getTotalParameterCount() { return count; } + + @Override + public Map getEntityColAndTypeBySubmittedNameMap() { + return ALLOWED_SORT_AND_FILTER_FIELDS; + } } diff --git a/src/main/java/org/brapi/test/BrAPITestServer/model/dto/EntityType.java b/src/main/java/org/brapi/test/BrAPITestServer/model/dto/EntityType.java index e69aa988..8f1156a8 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/model/dto/EntityType.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/model/dto/EntityType.java @@ -3,6 +3,8 @@ public enum EntityType { TEXT, UUID, - BOOLEAN + BOOLEAN, + // Sorting is done at the database schema/formula level for numbers. See GermplasmEntity.maleParentGid. No changes to filtering required for this datatype. + NUMBER // Add any other entity data types we should filter different in SearchQueryBuilder here } diff --git a/src/main/java/org/brapi/test/BrAPITestServer/model/entity/germ/GermplasmEntity.java b/src/main/java/org/brapi/test/BrAPITestServer/model/entity/germ/GermplasmEntity.java index 8b464b85..821c2873 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/model/entity/germ/GermplasmEntity.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/model/entity/germ/GermplasmEntity.java @@ -15,6 +15,7 @@ import org.brapi.test.BrAPITestServer.model.entity.pheno.TaxonEntity; import org.hibernate.annotations.BatchSize; +import org.hibernate.annotations.Formula; import org.hibernate.annotations.Where; import java.util.ArrayList; @@ -100,6 +101,24 @@ public class GermplasmEntity extends BrAPIPrimaryEntity { @ManyToOne(cascade = CascadeType.DETACH, fetch = FetchType.LAZY) private ProgramEntity program; + @Formula("(additional_info #>> '{createdDate}')") + private String createdDate; + + @Formula("(additional_info #>> '{createdBy,userName}')") + private String createdBy; + + @Formula("(additional_info #>> '{breedingMethod}')") + private String methodName; + + @Formula("cast(additional_info #>> '{femaleParentGid}' AS INTEGER)") + private Integer femaleParentGid; + + @Formula("cast(additional_info #>> '{maleParentGid}' AS INTEGER)") + private Integer maleParentGid; + + @Formula("cast(additional_info #>> '{importEntryNumber}' AS INTEGER)") + private Integer importEntryNumber; + public GermplasmInstituteEntity getHostInstitute() { if (getInstitutes() != null) { for (GermplasmInstituteEntity institute : getInstitutes()) { diff --git a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java index d18a30a0..d70faf7c 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java @@ -46,12 +46,7 @@ public String getQuery() { } public String getIdQuery() { - if (sortClause.isEmpty()) { - // By default, sort on entity id to have query result remain idempotent - sortClause = defaultSort; - } - - return selectOnlyIds + whereClause + sortClause; + return selectOnlyIds + whereClause; } public Map getParams() { @@ -150,7 +145,8 @@ public SearchQueryBuilder appendLike(String like, String columnName) { return this; } - public SearchQueryBuilder appendLikeIDs(String like, String columnName) { + // Used to convert non-string fields to string and use a like filter comparison + public SearchQueryBuilder appendLikeString(String like, String columnName) { String paramName = paramFilterPattern(columnName); if (like != null) { @@ -290,6 +286,8 @@ public SearchQueryBuilder join(String join, String name) { this.selectClause += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; this.selectOnlyIds += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; this.joinedTables.add(join); + } else if (joinedFetchedTables.contains(join) && !this.joinedTables.contains(join)) { + this.selectOnlyIds += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; } return this; } @@ -416,6 +414,12 @@ public SearchQueryBuilder sortBy(List sortBy, Map resetSortClause() { + this.sortClause = ""; + return this; + } + private void buildSort(SortBy sort) { this.sortClause += entityPrefix(sort.getSortedOn()) + " " + sort.getSortOrder() + " "; } @@ -446,8 +450,8 @@ public SearchQueryBuilder filterBy(List filterBy, Map findGermplasm(@Valid GermplasmSearchRequest request, Meta return germplasms; } - public List findGermplasmWithoutPaging(@Valid GermplasmSearchRequest request) { + public List findGermplasmWithoutPaging(@Valid GermplasmSearchRequest request) throws BrAPIServerException { List entities = findGermplasmEntitiesWithoutPaging(request); return entities.stream().map(this::convertFromEntity).collect(Collectors.toList()); } @@ -159,7 +159,7 @@ public Page findGermplasmEntities(@Valid GermplasmSearchRequest } // TODO: Investigate/consider removing this code. It is largely unused by DeltaBreed now, as paginating is the least memory exhaustive option. [BI-3020] - public List findGermplasmEntitiesWithoutPaging(@Valid GermplasmSearchRequest request) { + public List findGermplasmEntitiesWithoutPaging(@Valid GermplasmSearchRequest request) throws BrAPIServerException { SearchQueryBuilder searchQuery = buildGermplasmSearchQuery(request); List germs = germplasmRepository.findAllBySearch(searchQuery); @@ -170,7 +170,7 @@ public List findGermplasmEntitiesWithoutPaging(@Valid Germplasm return germs; } - private SearchQueryBuilder buildGermplasmSearchQuery(GermplasmSearchRequest request) { + private SearchQueryBuilder buildGermplasmSearchQuery(GermplasmSearchRequest request) throws BrAPIServerException { SearchQueryBuilder searchQuery = new SearchQueryBuilder( GermplasmEntity.class); searchQuery.leftJoinFetch("synonyms", "synonyms") @@ -207,7 +207,9 @@ private SearchQueryBuilder buildGermplasmSearchQuery(GermplasmS // .appendList(request.getProgenyDbIds(), "*progeny.germplasmDbId") .appendList(request.getGenus(), "genus").appendList(request.getSpecies(), "species") .appendNamesList(request.getBinomialNames(), "genus", "genus", "species") - .appendList(request.getFamilyCodes(), "familyCode"); + .appendList(request.getFamilyCodes(), "familyCode") + .filterBy(request.getFilterBy(), request.getEntityColAndTypeBySubmittedNameMap()) + .sortBy(request.getSortByElements(), request.getEntityColAndTypeBySubmittedNameMap()); return searchQuery; } @@ -339,7 +341,8 @@ private void fetchRemainingGermCollectionsUsingQuery(SearchQueryBuilder Date: Mon, 28 Sep 2026 17:34:09 -0400 Subject: [PATCH 2/5] Address issues with removing sorting from getIdQuery --- .../service/SearchQueryBuilder.java | 36 ++++++++++++++++--- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java index d70faf7c..ad6f4a36 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java @@ -18,6 +18,7 @@ public class SearchQueryBuilder { private String selectClause; private String selectOnlyIds; + private String selectOnlyIdsJoinsAndFilters; private String whereClause; private String defaultSort; private String sortClause; @@ -28,7 +29,8 @@ public class SearchQueryBuilder { public SearchQueryBuilder(Class clazz) { this.selectClause = "SELECT distinct entity FROM " + clazz.getSimpleName() + " entity "; - this.selectOnlyIds = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity "; + this.selectOnlyIds = "SELECT entity.id FROM " + clazz.getSimpleName() + " entity "; + this.selectOnlyIdsJoinsAndFilters = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity2 "; this.whereClause = "WHERE 1=1 "; this.defaultSort = " ORDER BY entity.id ASC "; this.sortClause = ""; @@ -45,8 +47,34 @@ public String getQuery() { return selectClause + whereClause + sortClause; } + /** + * The goal of this query is to return a query that will return only the BrAPI dbIds of the source entity, applying + * sorting, filtering, and pagination to the query. + * + * This has to be done in a very deliberate way because of both HQL and postgres constraints, so this query should + * end up looking something like this: + * + * SELECT entity.id + * FROM GermplasmEntity entity + * WHERE entity.id IN ( + * SELECT DISTINCT entity2.id + * FROM GermplasmEntity entity2 + * JOIN ... + * WHERE ... + * ) + * ORDER BY entity.germplasmName + * + * By containing the filtering and the joining in a subquery with a DISTINCT, we can control potential duplicates from breaking through the cracks. + * Then we can apply sorting outside of this query without a distinct to avoid postgres's requirement to include every order by column in the select clause. + * Distinct is not an issue here, since we are selecting only on id, which is always unique. + */ public String getIdQuery() { - return selectOnlyIds + whereClause; + if (sortClause.isEmpty()) { + // By default, sort on entity id to have query result remain idempotent + sortClause = defaultSort; + } + + return selectOnlyIds + " WHERE entity.id IN (" + selectOnlyIdsJoinsAndFilters + whereClause + ") " + sortClause; } public Map getParams() { @@ -284,10 +312,10 @@ public SearchQueryBuilder join(String join, String name) { if (!this.joinedTables.contains(join) && !this.joinedFetchedTables.contains(join)) { this.selectClause += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; - this.selectOnlyIds += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; + this.selectOnlyIdsJoinsAndFilters += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; this.joinedTables.add(join); } else if (joinedFetchedTables.contains(join) && !this.joinedTables.contains(join)) { - this.selectOnlyIds += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; + this.selectOnlyIdsJoinsAndFilters += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; } return this; } From a22c0a399083635bac879df42cedaaa649c7bfe9 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Tue, 29 Sep 2026 18:06:12 -0400 Subject: [PATCH 3/5] Removed exref ids/source, germplasm names from async check, add comment --- .../io/swagger/model/germ/GermplasmSearchRequest.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java b/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java index d1acd966..838a089c 100644 --- a/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java +++ b/src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java @@ -596,6 +596,8 @@ private String toIndentedString(java.lang.Object o) { @Override @JsonIgnore + // TODO: NOTE - External Reference IDs, exref sources, germplasmNames and germplasmDbIds have been removed from the parameter count to speed up lookups and prevent async search requests. + // TODO: We would need to reconsider this if we decide to use parameter count as the way to trigger async requests when complete [BI-3021] public Integer getTotalParameterCount() { Integer count = 0; if (this.accessionNumbers != null) @@ -606,16 +608,10 @@ public Integer getTotalParameterCount() { count += this.collections.size(); if (this.commonCropNames != null) count += this.commonCropNames.size(); - if (this.externalReferenceIds != null) - count += this.externalReferenceIds.size(); - if (this.externalReferenceSources != null) - count += this.externalReferenceSources.size(); if (this.familyCodes != null) count += this.familyCodes.size(); if (this.genus != null) count += this.genus.size(); - if (this.germplasmNames != null) - count += this.germplasmNames.size(); if (this.germplasmPUIs != null) count += this.germplasmPUIs.size(); if (this.instituteCodes != null) From 1c209985139017bbc8d3b20db2fcd70ec8488838 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Wed, 30 Sep 2026 14:14:43 -0400 Subject: [PATCH 4/5] Ensure subquery uses proper alias, add more clarifying comments --- .../service/SearchQueryBuilder.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java index ad6f4a36..c40669d0 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java @@ -18,7 +18,7 @@ public class SearchQueryBuilder { private String selectClause; private String selectOnlyIds; - private String selectOnlyIdsJoinsAndFilters; + private String selectOnlyIdsSubquery; private String whereClause; private String defaultSort; private String sortClause; @@ -29,8 +29,10 @@ public class SearchQueryBuilder { public SearchQueryBuilder(Class clazz) { this.selectClause = "SELECT distinct entity FROM " + clazz.getSimpleName() + " entity "; + // This is the top level query for selectOnlyIds. It is used to construct the full id query in getIdQuery() this.selectOnlyIds = "SELECT entity.id FROM " + clazz.getSimpleName() + " entity "; - this.selectOnlyIdsJoinsAndFilters = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity2 "; + // This subquery will contain all the joins and filters necessary for the selectOnlyIds query and protects against duplicates using distinct + this.selectOnlyIdsSubquery = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity "; this.whereClause = "WHERE 1=1 "; this.defaultSort = " ORDER BY entity.id ASC "; this.sortClause = ""; @@ -57,12 +59,14 @@ public String getQuery() { * SELECT entity.id * FROM GermplasmEntity entity * WHERE entity.id IN ( - * SELECT DISTINCT entity2.id + * SELECT distinct entity2.id * FROM GermplasmEntity entity2 - * JOIN ... - * WHERE ... - * ) - * ORDER BY entity.germplasmName + * JOIN entity2.externalReferences externalReference + * WHERE 1=1 AND externalReference.externalReferenceId in :externalReferenceId + * AND externalReference.externalReferenceSource in :externalReferenceSource + * AND entity2.program.id in :program_id + * ) + * ORDER BY entity.id ASC * * By containing the filtering and the joining in a subquery with a DISTINCT, we can control potential duplicates from breaking through the cracks. * Then we can apply sorting outside of this query without a distinct to avoid postgres's requirement to include every order by column in the select clause. @@ -74,7 +78,10 @@ public String getIdQuery() { sortClause = defaultSort; } - return selectOnlyIds + " WHERE entity.id IN (" + selectOnlyIdsJoinsAndFilters + whereClause + ") " + sortClause; + // This allows us to apply all the same filters/joins built up on the original entity with the inner entity. + String fullIdsSubquery = (selectOnlyIdsSubquery + whereClause).replace("entity", "entity2"); + + return selectOnlyIds + " WHERE entity.id IN (" + fullIdsSubquery + ") " + sortClause; } public Map getParams() { @@ -312,10 +319,10 @@ public SearchQueryBuilder join(String join, String name) { if (!this.joinedTables.contains(join) && !this.joinedFetchedTables.contains(join)) { this.selectClause += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; - this.selectOnlyIdsJoinsAndFilters += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; + this.selectOnlyIdsSubquery += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; this.joinedTables.add(join); } else if (joinedFetchedTables.contains(join) && !this.joinedTables.contains(join)) { - this.selectOnlyIdsJoinsAndFilters += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; + this.selectOnlyIdsSubquery += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " "; } return this; } From af9a82baf65186bb49a0485c1ff11fccb666afb6 Mon Sep 17 00:00:00 2001 From: Jason Loux Date: Mon, 5 Oct 2026 15:30:59 -0400 Subject: [PATCH 5/5] Implement regex expression for entity replacement for more fine grained usage --- .../BrAPITestServer/service/SearchQueryBuilder.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java index c40669d0..a7d5b3f0 100644 --- a/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java +++ b/src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java @@ -32,7 +32,7 @@ public SearchQueryBuilder(Class clazz) { // This is the top level query for selectOnlyIds. It is used to construct the full id query in getIdQuery() this.selectOnlyIds = "SELECT entity.id FROM " + clazz.getSimpleName() + " entity "; // This subquery will contain all the joins and filters necessary for the selectOnlyIds query and protects against duplicates using distinct - this.selectOnlyIdsSubquery = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity "; + this.selectOnlyIdsSubquery = "SELECT distinct entity2.id FROM " + clazz.getSimpleName() + " entity2 "; this.whereClause = "WHERE 1=1 "; this.defaultSort = " ORDER BY entity.id ASC "; this.sortClause = ""; @@ -78,9 +78,16 @@ public String getIdQuery() { sortClause = defaultSort; } - // This allows us to apply all the same filters/joins built up on the original entity with the inner entity. - String fullIdsSubquery = (selectOnlyIdsSubquery + whereClause).replace("entity", "entity2"); + // Now build the subquery to apply all the same filters/joins built up on the original entity with the inner entity. + // To do this, create a regex expression which can be used to identity all instances of an "entity" without any proceeding + // words, dots, underscores, or colons, and ignore any existing instances of entity2, which exists already in selectOnlyIdsSubquery + String entityRegex = "(?