From 41f4881363b8bbd6b06dca50fbf1ea830e582691 Mon Sep 17 00:00:00 2001 From: Laura Schanno Date: Thu, 27 Jul 2023 16:07:24 -0400 Subject: [PATCH] Move #NO_EXPANSION query test to separate class (#1769) Currently query tests for the #NO_EXPANSION function are located in the GroupingTestWithModel class. Since #NO_EXPANSION is not directly related to grouping in terms of functionality, move the #NO_EXPANSION tests to a separate test suite for clarity. --- .../transformer/GroupingTestWithModel.java | 96 ------- .../query/transformer/NoExpansionTests.java | 235 ++++++++++++++++++ 2 files changed, 235 insertions(+), 96 deletions(-) create mode 100644 warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTestWithModel.java b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTestWithModel.java index 9769912f095..9267a183bf2 100644 --- a/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTestWithModel.java +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/GroupingTestWithModel.java @@ -56,20 +56,17 @@ import datawave.helpers.PrintUtility; import datawave.ingest.data.TypeRegistry; import datawave.marking.MarkingFunctions; -import datawave.query.QueryParameters; import datawave.query.QueryTestTableHelper; import datawave.query.RebuildingScannerTestHelper; import datawave.query.attributes.Attribute; import datawave.query.common.grouping.GroupingUtil.GroupCountingHashMap; import datawave.query.common.grouping.GroupingUtil.GroupingTypeAttribute; import datawave.query.function.deserializer.KryoDocumentDeserializer; -import datawave.query.jexl.JexlASTHelper; import datawave.query.language.parser.jexl.JexlControlledQueryParser; import datawave.query.language.parser.jexl.LuceneToJexlQueryParser; import datawave.query.tables.ShardQueryLogic; import datawave.query.tables.edge.DefaultEdgeEventQueryLogic; import datawave.query.util.VisibilityWiseGuysIngestWithModel; -import datawave.test.JexlNodeAssert; import datawave.util.TableName; import datawave.webservice.edgedictionary.RemoteEdgeDictionary; import datawave.webservice.query.QueryImpl; @@ -118,18 +115,6 @@ protected BaseQueryResponse runTestQueryWithGrouping(Map expecte PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); return super.runTestQueryWithGrouping(expected, querystr, startDate, endDate, extraParms, client); } - - @Override - protected void runTestToComparePlans(String query, String expectedPlan, Date startDate, Date endDate, Map extraParms, - RebuildingScannerTestHelper.TEARDOWN teardown, RebuildingScannerTestHelper.INTERRUPT interrupt) throws Exception { - QueryTestTableHelper qtth = new QueryTestTableHelper(ShardRange.class.getName(), log, teardown, interrupt); - AccumuloClient client = qtth.client; - VisibilityWiseGuysIngestWithModel.writeItAll(client, VisibilityWiseGuysIngestWithModel.WhatKindaRange.SHARD); - PrintUtility.printTable(client, auths, TableName.SHARD); - PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); - PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); - super.runTestToComparePlans(query, expectedPlan, startDate, endDate, extraParms, client); - } } @RunWith(Arquillian.class) @@ -147,18 +132,6 @@ protected BaseQueryResponse runTestQueryWithGrouping(Map expecte PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); return super.runTestQueryWithGrouping(expected, querystr, startDate, endDate, extraParms, client); } - - @Override - protected void runTestToComparePlans(String query, String expectedPlan, Date startDate, Date endDate, Map extraParms, - RebuildingScannerTestHelper.TEARDOWN teardown, RebuildingScannerTestHelper.INTERRUPT interrupt) throws Exception { - QueryTestTableHelper qtth = new QueryTestTableHelper(ShardRange.class.getName(), log, teardown, interrupt); - AccumuloClient client = qtth.client; - VisibilityWiseGuysIngestWithModel.writeItAll(client, VisibilityWiseGuysIngestWithModel.WhatKindaRange.DOCUMENT); - PrintUtility.printTable(client, auths, TableName.SHARD); - PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); - PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); - super.runTestToComparePlans(query, expectedPlan, startDate, endDate, extraParms, client); - } } @Rule @@ -284,36 +257,6 @@ protected BaseQueryResponse runTestQueryWithGrouping(Map expecte return response; } - protected abstract void runTestToComparePlans(String query, String expectedPlan, Date startDate, Date endDate, Map extraParms, - RebuildingScannerTestHelper.TEARDOWN teardown, RebuildingScannerTestHelper.INTERRUPT interrupt) throws Exception; - - protected void runTestToComparePlans(String query, String expectedPlan, Date startDate, Date endDate, Map extraParms, AccumuloClient client) - throws Exception { - log.debug("test plan against expected plan"); - - QueryImpl settings = new QueryImpl(); - settings.setBeginDate(startDate); - settings.setEndDate(endDate); - settings.setPagesize(Integer.MAX_VALUE); - settings.setQueryAuthorizations(auths.serialize()); - settings.setQuery(query); - settings.setParameters(extraParms); - settings.setId(UUID.randomUUID()); - - log.debug("query: " + settings.getQuery()); - log.debug("logic: " + settings.getQueryLogicName()); - - GenericQueryConfiguration config = logic.initialize(client, settings, authSet); - logic.setupQuery(config); - - String plan = logic.getPlan(client, settings, authSet, true, true); - - // order of terms in planned script is arbitrary, fall back to comparing the jexl trees - ASTJexlScript plannedScript = JexlASTHelper.parseJexlQuery(plan); - ASTJexlScript expectedScript = JexlASTHelper.parseJexlQuery(expectedPlan); - JexlNodeAssert.assertThat(expectedScript).isEqualTo(plannedScript); - } - @Test public void testGrouping() throws Exception { Map extraParameters = new HashMap<>(); @@ -588,43 +531,4 @@ public void testCountingMapAgain() { } } } - - @Test - public void testNoExpansion() throws Exception { - Map extraParameters = new HashMap<>(); - - Date startDate = format.parse("20091231"); - Date endDate = format.parse("20150101"); - - // base case with default query model expansion - String queryString = "COLOR == 'blue'"; - String expectedPlan = "(COLOR == 'blue' || HUE == 'blue')"; - runTestToComparePlans(queryString, expectedPlan, startDate, endDate, extraParameters, NEVER, RebuildingScannerTestHelper.INTERRUPT.NEVER); - - // case where #NO_EXPANSION is specified in the query string itself - queryString = "COLOR == 'blue' && f:noExpansion(COLOR)"; - expectedPlan = "COLOR == 'blue'"; - runTestToComparePlans(queryString, expectedPlan, startDate, endDate, extraParameters, NEVER, RebuildingScannerTestHelper.INTERRUPT.NEVER); - - // case where #NO_EXPANSION is specified via query parameters only - extraParameters.put(QueryParameters.NO_EXPANSION_FIELDS, "COLOR"); - queryString = "COLOR == 'blue'"; - expectedPlan = "COLOR == 'blue'"; - runTestToComparePlans(queryString, expectedPlan, startDate, endDate, extraParameters, NEVER, RebuildingScannerTestHelper.INTERRUPT.NEVER); - - // case where #NO_EXPANSION is specified in both query parameters and the query string - extraParameters.clear(); - extraParameters.put(QueryParameters.NO_EXPANSION_FIELDS, "COLOR"); - queryString = "COLOR == 'blue' && f:noExpansion(COLOR)"; - expectedPlan = "COLOR == 'blue'"; - runTestToComparePlans(queryString, expectedPlan, startDate, endDate, extraParameters, NEVER, RebuildingScannerTestHelper.INTERRUPT.NEVER); - - // case where #NO_EXPANSION in query params is the wrong field and query string function is the correct one - extraParameters.clear(); - extraParameters.put(QueryParameters.NO_EXPANSION_FIELDS, "HUE"); - queryString = "COLOR == 'blue' && f:noExpansion(COLOR)"; - expectedPlan = "COLOR == 'blue'"; - runTestToComparePlans(queryString, expectedPlan, startDate, endDate, extraParameters, NEVER, RebuildingScannerTestHelper.INTERRUPT.NEVER); - } - } diff --git a/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java b/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java new file mode 100644 index 00000000000..2b43678187a --- /dev/null +++ b/warehouse/query-core/src/test/java/datawave/query/transformer/NoExpansionTests.java @@ -0,0 +1,235 @@ +package datawave.query.transformer; + +import java.text.DateFormat; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; +import java.util.TimeZone; +import java.util.UUID; + +import javax.inject.Inject; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.security.Authorizations; +import org.apache.commons.jexl2.parser.ASTJexlScript; +import org.apache.log4j.Logger; +import org.jboss.arquillian.container.test.api.Deployment; +import org.jboss.arquillian.junit.Arquillian; +import org.jboss.shrinkwrap.api.ShrinkWrap; +import org.jboss.shrinkwrap.api.asset.StringAsset; +import org.jboss.shrinkwrap.api.spec.JavaArchive; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import datawave.configuration.spring.SpringBean; +import datawave.helpers.PrintUtility; +import datawave.ingest.data.TypeRegistry; +import datawave.query.QueryParameters; +import datawave.query.QueryTestTableHelper; +import datawave.query.RebuildingScannerTestHelper; +import datawave.query.function.deserializer.KryoDocumentDeserializer; +import datawave.query.jexl.JexlASTHelper; +import datawave.query.tables.ShardQueryLogic; +import datawave.query.tables.edge.DefaultEdgeEventQueryLogic; +import datawave.query.util.VisibilityWiseGuysIngestWithModel; +import datawave.test.JexlNodeAssert; +import datawave.util.TableName; +import datawave.webservice.edgedictionary.RemoteEdgeDictionary; +import datawave.webservice.query.QueryImpl; +import datawave.webservice.query.configuration.GenericQueryConfiguration; + +/** + * Tests for usage of #NO_EXPANSION in queries. + */ +public abstract class NoExpansionTests { + + @RunWith(Arquillian.class) + public static class ShardRange extends NoExpansionTests { + + @Override + protected VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange() { + return VisibilityWiseGuysIngestWithModel.WhatKindaRange.SHARD; + } + } + + @RunWith(Arquillian.class) + public static class DocumentRange extends NoExpansionTests { + + @Override + protected VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange() { + return VisibilityWiseGuysIngestWithModel.WhatKindaRange.DOCUMENT; + } + } + + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + private static final Logger log = Logger.getLogger(NoExpansionTests.class); + private static final Authorizations auths = new Authorizations("ALL", "E", "I"); + + @Inject + @SpringBean(name = "EventQuery") + protected ShardQueryLogic logic; + protected Set authSet = Collections.singleton(auths); + protected KryoDocumentDeserializer deserializer; + + private final DateFormat format = new SimpleDateFormat("yyyyMMdd"); + private final Map queryParameters = new HashMap<>(); + private Date startDate; + private Date endDate; + private String query; + private String expectedPlan; + + @Deployment + public static JavaArchive createDeployment() { + return ShrinkWrap.create(JavaArchive.class) + .addPackages(true, "org.apache.deltaspike", "io.astefanutti.metrics.cdi", "datawave.query", "org.jboss.logging", + "datawave.webservice.query.result.event") + .deleteClass(DefaultEdgeEventQueryLogic.class).deleteClass(RemoteEdgeDictionary.class) + .deleteClass(datawave.query.metrics.QueryMetricQueryLogic.class).deleteClass(datawave.query.metrics.ShardTableQueryMetricHandler.class) + .addAsManifestResource(new StringAsset( + "" + "datawave.query.tables.edge.MockAlternative" + ""), + "beans.xml"); + } + + @AfterClass + public static void teardown() { + TypeRegistry.reset(); + } + + @Before + public void setup() throws ParseException { + TimeZone.setDefault(TimeZone.getTimeZone("GMT")); + + this.logic.setFullTableScanEnabled(true); + this.logic.setMaxEvaluationPipelines(1); + this.logic.setQueryExecutionForPageTimeout(300000000000000L); + this.deserializer = new KryoDocumentDeserializer(); + this.startDate = format.parse("20091231"); + this.endDate = format.parse("20150101"); + } + + @After + public void tearDown() { + queryParameters.clear(); + } + + protected abstract VisibilityWiseGuysIngestWithModel.WhatKindaRange getRange(); + + private void runTestQuery() throws Exception { + log.debug("test plan against expected plan"); + + QueryImpl settings = new QueryImpl(); + settings.setBeginDate(this.startDate); + settings.setEndDate(this.endDate); + settings.setPagesize(Integer.MAX_VALUE); + settings.setQueryAuthorizations(auths.serialize()); + settings.setQuery(this.query); + settings.setParameters(this.queryParameters); + settings.setId(UUID.randomUUID()); + + log.debug("query: " + settings.getQuery()); + log.debug("logic: " + settings.getQueryLogicName()); + + AccumuloClient client = createClient(); + GenericQueryConfiguration config = logic.initialize(client, settings, authSet); + logic.setupQuery(config); + + String plan = logic.getPlan(client, settings, authSet, true, true); + + // order of terms in planned script is arbitrary, fall back to comparing the jexl trees + ASTJexlScript plannedScript = JexlASTHelper.parseJexlQuery(plan); + ASTJexlScript expectedScript = JexlASTHelper.parseJexlQuery(this.expectedPlan); + JexlNodeAssert.assertThat(expectedScript).isEqualTo(plannedScript); + } + + private AccumuloClient createClient() throws Exception { + AccumuloClient client = new QueryTestTableHelper(getClass().getName(), log, RebuildingScannerTestHelper.TEARDOWN.NEVER, + RebuildingScannerTestHelper.INTERRUPT.NEVER).client; + VisibilityWiseGuysIngestWithModel.writeItAll(client, getRange()); + PrintUtility.printTable(client, auths, TableName.SHARD); + PrintUtility.printTable(client, auths, TableName.SHARD_INDEX); + PrintUtility.printTable(client, auths, QueryTestTableHelper.MODEL_TABLE_NAME); + return client; + } + + private void givenQueryParameter(String key, String value) { + this.queryParameters.put(key, value); + } + + private void givenQuery(String query) { + this.query = query; + } + + private void givenExpectedPlan(String expectedPlan) { + this.expectedPlan = expectedPlan; + } + + /** + * Base test to verify expansion happens by default. + */ + @Test + public void testDefaultQueryModelExpansion() throws Exception { + givenQuery("COLOR == 'blue'"); + givenExpectedPlan("(COLOR == 'blue' || HUE == 'blue')"); + + runTestQuery(); + } + + /** + * Verify that when #NO_EXPANSION is specified in the query string itself, expansion does not occur. + */ + @Test + public void testNoExpansionViaFunction() throws Exception { + givenQuery("COLOR == 'blue' && f:noExpansion(COLOR)"); + givenExpectedPlan("COLOR == 'blue'"); + + runTestQuery(); + } + + /** + * Verify that when #NO_EXPANSION is specified via the query parameters, expansion does not occur. + */ + @Test + public void testNoExpansionViaQueryParameters() throws Exception { + givenQuery("COLOR == 'blue'"); + givenQueryParameter(QueryParameters.NO_EXPANSION_FIELDS, "COLOR"); + givenExpectedPlan("COLOR == 'blue'"); + + runTestQuery(); + } + + /** + * Verify that when #NO_EXPANSION is specified in the query string itself and in query parameters, expansion does not occur. + */ + @Test + public void testNoExpansionViaFunctionAndQueryParameters() throws Exception { + givenQuery("COLOR == 'blue' && f:noExpansion(COLOR)"); + givenQueryParameter(QueryParameters.NO_EXPANSION_FIELDS, "COLOR"); + givenExpectedPlan("COLOR == 'blue'"); + + runTestQuery(); + } + + /** + * Verify that when #NO_EXPANSION is specified with the correct field in the query string, and the wrong field in the query parameters, that the correct + * field is retained. + */ + @Test + public void testConflictingNoExpansionFields() throws Exception { + givenQuery("COLOR == 'blue' && f:noExpansion(COLOR)"); + givenQueryParameter(QueryParameters.NO_EXPANSION_FIELDS, "HUE"); + givenExpectedPlan("COLOR == 'blue'"); + + runTestQuery(); + } +}