Changes to support filtering/sorting for germplasm

src/main/java/io/swagger/model/germ/GermplasmSearchRequest.java

@@ -3,12 +3,34 @@
 import com.fasterxml.jackson.annotation.JsonIgnore;
 import com.fasterxml.jackson.annotation.JsonProperty;
 import io.swagger.model.SearchRequest;
+import org.brapi.test.BrAPITestServer.model.dto.EntityColumnNameAndType;
+import org.brapi.test.BrAPITestServer.model.dto.EntityType;
 
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 import java.util.Objects;
 
 public class GermplasmSearchRequest extends SearchRequest {
+
+	// Key - allowed sort or field filter name for this entity
+	// Value = entity field name and type that represents the submitted field. Used later on in query building.
+	private static final Map<String, EntityColumnNameAndType> ALLOWED_SORT_AND_FILTER_FIELDS =
+			Map.ofEntries(
+					// accessionNumber = GID for DeltaBreed
+					Map.entry("accessionNumber", new EntityColumnNameAndType("accessionNumber", EntityType.NUMBER)),
+					Map.entry("defaultDisplayName", new EntityColumnNameAndType("defaultDisplayName", EntityType.TEXT)),
+					Map.entry("breedingMethod", new EntityColumnNameAndType("methodName", EntityType.TEXT)),
+					Map.entry("seedSource", new EntityColumnNameAndType("seedSource", EntityType.TEXT)),
+					Map.entry("pedigree",new EntityColumnNameAndType("pedigree.pedigreeString", EntityType.TEXT)),
+					Map.entry("femaleParentGID", new EntityColumnNameAndType("femaleParentGid", EntityType.NUMBER)),
+					Map.entry("maleParentGID", new EntityColumnNameAndType("maleParentGid", EntityType.NUMBER)),
+					Map.entry("createdDate", new EntityColumnNameAndType("createdDate",  EntityType.TEXT)),
+					Map.entry("createdBy", new EntityColumnNameAndType("createdBy", EntityType.TEXT)),
+					Map.entry("importEntryNumber", new EntityColumnNameAndType("importEntryNumber", EntityType.NUMBER)),
+					Map.entry("synonyms", new EntityColumnNameAndType("*synonyms.synonym", EntityType.TEXT))
+			);
+
 	@JsonProperty("accessionNumbers")
 	private List<String> accessionNumbers = null;
 
@@ -574,6 +596,8 @@ private String toIndentedString(java.lang.Object o) {
 
 	@Override
 	@JsonIgnore
+	// TODO: NOTE - External Reference IDs, exref sources, germplasmNames and germplasmDbIds have been removed from the parameter count to speed up lookups and prevent async search requests.
+	// TODO: We would need to reconsider this if we decide to use parameter count as the way to trigger async requests when complete [BI-3021]
 	public Integer getTotalParameterCount() {
 		Integer count = 0;
 		if (this.accessionNumbers != null)
@@ -584,16 +608,10 @@ public Integer getTotalParameterCount() {
 			count += this.collections.size();
 		if (this.commonCropNames != null)
 			count += this.commonCropNames.size();
-		if (this.externalReferenceIds != null)
-			count += this.externalReferenceIds.size();
-		if (this.externalReferenceSources != null)
-			count += this.externalReferenceSources.size();
 		if (this.familyCodes != null)
 			count += this.familyCodes.size();
 		if (this.genus != null)
 			count += this.genus.size();
-		if (this.germplasmNames != null)
-			count += this.germplasmNames.size();
 		if (this.germplasmPUIs != null)
 			count += this.germplasmPUIs.size();
 		if (this.instituteCodes != null)
@@ -621,4 +639,9 @@ public Integer getTotalParameterCount() {
 
 		return count;
 	}
+
+	@Override
+	public Map<String, EntityColumnNameAndType> getEntityColAndTypeBySubmittedNameMap() {
+		return ALLOWED_SORT_AND_FILTER_FIELDS;
+	}
 }

src/main/java/org/brapi/test/BrAPITestServer/model/dto/EntityType.java

@@ -3,6 +3,8 @@
 public enum EntityType {
     TEXT,
     UUID,
-    BOOLEAN
+    BOOLEAN,
+    // Sorting is done at the database schema/formula level for numbers.  See GermplasmEntity.maleParentGid.  No changes to filtering required for this datatype.
+    NUMBER
     // Add any other entity data types we should filter different in SearchQueryBuilder here
 }

src/main/java/org/brapi/test/BrAPITestServer/model/entity/germ/GermplasmEntity.java

@@ -15,6 +15,7 @@
 import org.brapi.test.BrAPITestServer.model.entity.pheno.TaxonEntity;
 
 import org.hibernate.annotations.BatchSize;
+import org.hibernate.annotations.Formula;
 import org.hibernate.annotations.Where;
 
 import java.util.ArrayList;
@@ -100,6 +101,24 @@ public class GermplasmEntity extends BrAPIPrimaryEntity {
     @ManyToOne(cascade = CascadeType.DETACH, fetch = FetchType.LAZY)
     private ProgramEntity program;
 
+    @Formula("(additional_info #>> '{createdDate}')")
+    private String createdDate;
+
+    @Formula("(additional_info #>> '{createdBy,userName}')")
+    private String createdBy;
+
+    @Formula("(additional_info #>> '{breedingMethod}')")
+    private String methodName;
+
+    @Formula("cast(additional_info #>> '{femaleParentGid}' AS INTEGER)")
+    private Integer femaleParentGid;
+
+    @Formula("cast(additional_info #>> '{maleParentGid}' AS INTEGER)")
+    private Integer maleParentGid;
+
+    @Formula("cast(additional_info #>> '{importEntryNumber}' AS INTEGER)")
+    private Integer importEntryNumber;
+
     public GermplasmInstituteEntity getHostInstitute() {
         if (getInstitutes() != null) {
             for (GermplasmInstituteEntity institute : getInstitutes()) {

src/main/java/org/brapi/test/BrAPITestServer/service/SearchQueryBuilder.java

@@ -18,6 +18,7 @@ public class SearchQueryBuilder<T> {
 
 	private String selectClause;
 	private String selectOnlyIds;
+	private String selectOnlyIdsSubquery;
 	private String whereClause;
 	private String defaultSort;
 	private String sortClause;
@@ -28,7 +29,10 @@ public class SearchQueryBuilder<T> {
 
 	public SearchQueryBuilder(Class<T> clazz) {
 		this.selectClause = "SELECT distinct entity FROM " + clazz.getSimpleName() + " entity ";
-		this.selectOnlyIds = "SELECT distinct entity.id FROM " + clazz.getSimpleName() + " entity ";
+		// This is the top level query for selectOnlyIds.  It is used to construct the full id query in getIdQuery()
+		this.selectOnlyIds = "SELECT entity.id FROM " + clazz.getSimpleName() + " entity ";
+		// This subquery will contain all the joins and filters necessary for the selectOnlyIds query and protects against duplicates using distinct
+		this.selectOnlyIdsSubquery = "SELECT distinct entity2.id FROM " + clazz.getSimpleName() + " entity2 ";
 		this.whereClause = "WHERE 1=1 ";
 		this.defaultSort = " ORDER BY entity.id ASC ";
 		this.sortClause = "";
@@ -45,13 +49,46 @@ public String getQuery() {
 		return selectClause + whereClause + sortClause;
 	}
 
+	/**
+	 * The goal of this query is to return a query that will return only the BrAPI dbIds of the source entity, applying
+	 * sorting, filtering, and pagination to the query.
+	 *
+	 * This has to be done in a very deliberate way because of both HQL and postgres constraints, so this query should
+	 * end up looking something like this:
+	 *
+	 * SELECT entity.id
+	 * FROM GermplasmEntity entity
+	 * WHERE entity.id IN (
+	 *     SELECT distinct entity2.id
+	 *     FROM GermplasmEntity entity2
+	 *     JOIN entity2.externalReferences externalReference
+	 *     WHERE 1=1 AND externalReference.externalReferenceId in :externalReferenceId
+	 *     AND externalReference.externalReferenceSource in :externalReferenceSource
+	 *     AND entity2.program.id in :program_id
+	 *     )
+	 * ORDER BY entity.id ASC
+	 *
+	 * By containing the filtering and the joining in a subquery with a DISTINCT, we can control potential duplicates from breaking through the cracks.
+	 * Then we can apply sorting outside of this query without a distinct to avoid postgres's requirement to include every order by column in the select clause.
+	 * Distinct is not an issue here, since we are selecting only on id, which is always unique.
+	 */
 	public String getIdQuery() {
 		if (sortClause.isEmpty()) {
 			// By default, sort on entity id to have query result remain idempotent
 			sortClause = defaultSort;
 		}
 
-		return selectOnlyIds + whereClause + sortClause;
+		// Now build the subquery to apply all the same filters/joins built up on the original entity with the inner entity.
+
+		// To do this, create a regex expression which can be used to identity all instances of an "entity" without any proceeding
+		// words, dots, underscores, or colons, and ignore any existing instances of entity2, which exists already in selectOnlyIdsSubquery
+		String entityRegex = "(?<![\\\\w.:])entity(?!2)";
+
+		// Now put the subquery together with the where clause and apply the regex expression
+		String fullIdsSubquery = (selectOnlyIdsSubquery + whereClause).replaceAll(entityRegex, "entity2");
+
+		// Finally, apply outer expression and final where clause
+		return selectOnlyIds + " WHERE entity.id IN (" + fullIdsSubquery + ") " + sortClause;
 	}
 
 	public Map<String, Object> getParams() {
@@ -150,7 +187,8 @@ public SearchQueryBuilder<T> appendLike(String like, String columnName) {
 		return this;
 	}
 
-	public SearchQueryBuilder<T> appendLikeIDs(String like, String columnName) {
+	// Used to convert non-string fields to string and use a like filter comparison
+	public SearchQueryBuilder<T> appendLikeString(String like, String columnName) {
 		String paramName = paramFilterPattern(columnName);
 
 		if (like != null) {
@@ -288,8 +326,10 @@ public SearchQueryBuilder<T> join(String join, String name) {
 
 		if (!this.joinedTables.contains(join) && !this.joinedFetchedTables.contains(join)) {
 			this.selectClause += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " ";
-			this.selectOnlyIds += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " ";
+			this.selectOnlyIdsSubquery += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " ";
 			this.joinedTables.add(join);
+		} else if (joinedFetchedTables.contains(join) && !this.joinedTables.contains(join)) {
+			this.selectOnlyIdsSubquery += "JOIN " + entityPrefix(join) + " " + paramFilter(name) + " ";
 		}
 		return this;
 	}
@@ -416,6 +456,12 @@ public SearchQueryBuilder<T> sortBy(List<SortBy> sortBy, Map<String, EntityColum
 		return this;
 	}
 
+	// Used to continue utilizing the same search query, like in GermplasmService fetching without pagination use case
+	public SearchQueryBuilder<T> resetSortClause() {
+		this.sortClause = "";
+		return this;
+	}
+
 	private void buildSort(SortBy sort) {
 		this.sortClause += entityPrefix(sort.getSortedOn()) + " " + sort.getSortOrder() + " ";
 	}
@@ -446,8 +492,8 @@ public SearchQueryBuilder<T> filterBy(List<FilterBy> filterBy, Map<String, Entit
 
 			if (entityColumnNameAndType.getEntityType() == EntityType.TEXT) {
 				searchQuery = appendLike(filter.getValue().toLowerCase(), entityColumnNameAndType.getEntityColumnName());
-			} else if (entityColumnNameAndType.getEntityType() == EntityType.UUID) {
-				searchQuery = appendLikeIDs(filter.getValue(), entityColumnNameAndType.getEntityColumnName());
+			} else if (entityColumnNameAndType.getEntityType() == EntityType.UUID || entityColumnNameAndType.getEntityType() == EntityType.NUMBER) {
+				searchQuery = appendLikeString(filter.getValue(), entityColumnNameAndType.getEntityColumnName());
 			}
 		}
 

src/main/java/org/brapi/test/BrAPITestServer/service/germ/GermplasmService.java

@@ -117,7 +117,7 @@ public List<Germplasm> findGermplasm(@Valid GermplasmSearchRequest request, Meta
 		return germplasms;
 	}
 
-	public List<Germplasm> findGermplasmWithoutPaging(@Valid GermplasmSearchRequest request) {
+	public List<Germplasm> findGermplasmWithoutPaging(@Valid GermplasmSearchRequest request) throws BrAPIServerException {
 		List<GermplasmEntity> entities = findGermplasmEntitiesWithoutPaging(request);
 		return entities.stream().map(this::convertFromEntity).collect(Collectors.toList());
 	}
@@ -159,7 +159,7 @@ public Page<GermplasmEntity> findGermplasmEntities(@Valid GermplasmSearchRequest
 	}
 
 	// TODO: Investigate/consider removing this code.  It is largely unused by DeltaBreed now, as paginating is the least memory exhaustive option. [BI-3020]
-	public List<GermplasmEntity> findGermplasmEntitiesWithoutPaging(@Valid GermplasmSearchRequest request) {
+	public List<GermplasmEntity> findGermplasmEntitiesWithoutPaging(@Valid GermplasmSearchRequest request) throws BrAPIServerException {
 		SearchQueryBuilder<GermplasmEntity> searchQuery = buildGermplasmSearchQuery(request);
 
 		List<GermplasmEntity>  germs = germplasmRepository.findAllBySearch(searchQuery);
@@ -170,7 +170,7 @@ public List<GermplasmEntity> findGermplasmEntitiesWithoutPaging(@Valid Germplasm
 		return germs;
 	}
 
-	private SearchQueryBuilder<GermplasmEntity> buildGermplasmSearchQuery(GermplasmSearchRequest request) {
+	private SearchQueryBuilder<GermplasmEntity> buildGermplasmSearchQuery(GermplasmSearchRequest request) throws BrAPIServerException {
 		SearchQueryBuilder<GermplasmEntity> searchQuery = new SearchQueryBuilder<GermplasmEntity>(
 				GermplasmEntity.class);
 		searchQuery.leftJoinFetch("synonyms", "synonyms")
@@ -207,7 +207,9 @@ private SearchQueryBuilder<GermplasmEntity> buildGermplasmSearchQuery(GermplasmS
 				// .appendList(request.getProgenyDbIds(), "*progeny.germplasmDbId")
 				.appendList(request.getGenus(), "genus").appendList(request.getSpecies(), "species")
 				.appendNamesList(request.getBinomialNames(), "genus", "genus", "species")
-				.appendList(request.getFamilyCodes(), "familyCode");
+				.appendList(request.getFamilyCodes(), "familyCode")
+				.filterBy(request.getFilterBy(), request.getEntityColAndTypeBySubmittedNameMap())
+				.sortBy(request.getSortByElements(), request.getEntityColAndTypeBySubmittedNameMap());
 		return searchQuery;
 	}
 
@@ -339,7 +341,8 @@ private void fetchRemainingGermCollectionsUsingQuery(SearchQueryBuilder<Germplas
 				.removeLeftJoinFetch("breedingMethod", "breedingMethod")
 				.removeLeftJoinFetch("crop", "crop")
 				.removeLeftJoinFetch("pedigree", "pedigree")
-				.removeLeftJoinFetch("*pedigree.crossingProject", "crossingProject");
+				.removeLeftJoinFetch("*pedigree.crossingProject", "crossingProject")
+				.resetSortClause();
 		// Fetch xrefs
 		log.debug("Fetching xrefs");
 		searchQuery.leftJoinFetch("externalReferences",