diff --git a/cardinality_estimation.c b/cardinality_estimation.c index 8ab98f3c..f0cca328 100644 --- a/cardinality_estimation.c +++ b/cardinality_estimation.c @@ -81,8 +81,17 @@ predict_for_relation(List *clauses, List *selectivities, List *relsigns, &ncols, &features); data = OkNNr_allocate(ncols); - if (load_fss_ext(query_context.fspace_hash, *fss, data, NULL)) + if (load_aqo_data(query_context.fspace_hash, *fss, data, false) && + data->rows >= (aqo_predict_with_few_neighbors ? 1 : aqo_k)) result = OkNNr_predict(data, features); + /* Try to search in surrounding feature spaces for the same node */ + else if (use_wide_search && load_aqo_data(query_context.fspace_hash, *fss, data, true)) + { + elog(DEBUG5, "[AQO] Make prediction for fss "INT64_FORMAT" by a neighbour " + "includes %d feature(s) and %d fact(s).", + (int64) *fss, data->cols, data->rows); + result = OkNNr_predict(data, features); + } else { /* @@ -91,17 +100,7 @@ predict_for_relation(List *clauses, List *selectivities, List *relsigns, * small part of paths was used for AQO learning and stored into * the AQO knowledge base. */ - - /* Try to search in surrounding feature spaces for the same node */ - if (!load_aqo_data(query_context.fspace_hash, *fss, data, NULL, use_wide_search, features)) - result = -1; - else - { - elog(DEBUG5, "[AQO] Make prediction for fss %d by a neighbour " - "includes %d feature(s) and %d fact(s).", - *fss, data->cols, data->rows); - result = OkNNr_predict(data, features); - } + result = -1; } #ifdef AQO_DEBUG_PRINT diff --git a/cardinality_hooks.c b/cardinality_hooks.c index ae6dff5e..93fb73b1 100644 --- a/cardinality_hooks.c +++ b/cardinality_hooks.c @@ -414,7 +414,7 @@ predict_num_groups(PlannerInfo *root, Path *subpath, List *group_exprs, *fss = get_grouped_exprs_hash(child_fss, group_exprs); memset(&data, 0, sizeof(OkNNrdata)); - if (!load_fss_ext(query_context.fspace_hash, *fss, &data, NULL)) + if (!load_aqo_data(query_context.fspace_hash, *fss, &data, false)) return -1; Assert(data.rows == 1); diff --git a/expected/gucs.out b/expected/gucs.out index f33aa6b2..d083f6e2 100644 --- a/expected/gucs.out +++ b/expected/gucs.out @@ -145,4 +145,5 @@ SELECT count(*) FROM aqo_query_stat; 0 (1 row) +DROP TABLE t; DROP EXTENSION aqo; diff --git a/expected/look_a_like.out b/expected/look_a_like.out index 594f017e..854bb852 100644 --- a/expected/look_a_like.out +++ b/expected/look_a_like.out @@ -9,8 +9,9 @@ SELECT true AS success FROM aqo_reset(); SET aqo.wide_search = 'on'; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; -set aqo.show_hash = 'off'; +SET aqo.show_hash = 'off'; SET aqo.min_neighbors_for_predicting = 1; +SET aqo.predict_with_few_neighbors = 'off'; SET enable_nestloop = 'off'; SET enable_mergejoin = 'off'; SET enable_material = 'off'; @@ -553,9 +554,131 @@ WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT L JOINS: 2 (24 rows) +-- Next few test cases focus on fss corresponding to (x1 > ? AND x2 < ? AND x3 < ?). We will denote +-- it by fss0. At this moment there is exactly one fs with (fs, fss0, dbid) record in aqo_data. We'll +-- refer to it as fs0. +-- Let's create another fs for fss0. We'll call this fs fs1. Since aqo.wide_search='on', +-- aqo.min_neighbors_for_predicting=1, and there is (fs0, fss0, dbid) data record, AQO must be used here. +SELECT str AS result +FROM expln(' +SELECT * FROM A WHERE x1 > -100 AND x2 < 10 AND x3 < 10;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + result +---------------------------------------------------------------------- + Seq Scan on public.a (actual rows=100 loops=1) + AQO: rows=20, error=-400% + Output: x1, x2, x3 + Filter: ((a.x1 > '-100'::integer) AND (a.x2 < 10) AND (a.x3 < 10)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(7 rows) + +-- Now there are 2 data records for fss0: one for (fs0, fss0, dbid) and one for (fs1, fss0, dbid) +-- We repeat previous query, but set aqo.min_neighbors_for_predicting to 2. Since aqo.predict_with_few_neighbors +-- is 'off', AQO is obliged to use both data records for fss0. +SET aqo.min_neighbors_for_predicting = 2; +SELECT str AS result +FROM expln(' +SELECT * FROM A WHERE x1 > 1 AND x2 < 10 AND x3 < 10;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + result +-------------------------------------------------------- + Seq Scan on public.a (actual rows=80 loops=1) + AQO: rows=77, error=-4% + Output: x1, x2, x3 + Filter: ((a.x1 > 1) AND (a.x2 < 10) AND (a.x3 < 10)) + Rows Removed by Filter: 20 + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(8 rows) + +-- Now there are 3 data records for fss0: 1 for (fs0, fss0, dbid) and 2 for (fs1, fss0, dbid) +-- Lastly, we run invoke query with previously unseen fs with fss0 feature subspace. AQO must use +-- three data records from two neighbors for this one. +SET aqo.min_neighbors_for_predicting = 3; +SELECT str AS result +FROM expln(' +SELECT x2 FROM A WHERE x1 > 3 AND x2 < 10 AND x3 < 10 GROUP BY(x2);') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + result +-------------------------------------------------------------- + HashAggregate (actual rows=6 loops=1) + AQO not used + Output: x2 + Group Key: a.x2 + -> Seq Scan on public.a (actual rows=60 loops=1) + AQO: rows=71, error=15% + Output: x1, x2, x3 + Filter: ((a.x1 > 3) AND (a.x2 < 10) AND (a.x3 < 10)) + Rows Removed by Filter: 40 + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(12 rows) + +----- +DROP TABLE IF EXISTS t; +NOTICE: table "t" does not exist, skipping +CREATE TABLE t AS SELECT x, x AS y, x AS z FROM generate_series(1, 10000) x; +ANALYZE t; +SELECT true AS success FROM aqo_reset(); + success +--------- + t +(1 row) + +-- Test that when there are less records than aqo.min_neighbors_for_predicting for given (fs, fss, dbid) +-- and aqo.predict_with_few_neighbors is off, those records have higher precedence for cardinality estimation +-- than neighbors' records. +SELECT str AS result +FROM expln(' +select * from t where x <= 10000 and y <= 10000 and z <= 10000;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + result +------------------------------------------------------------------ + Seq Scan on public.t (actual rows=10000 loops=1) + AQO not used + Output: x, y, z + Filter: ((t.x <= 10000) AND (t.y <= 10000) AND (t.z <= 10000)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(7 rows) + +DO +$$ +BEGIN + for counter in 1..20 loop + EXECUTE format('explain analyze select *, 1 from t where x <= 1 and y <= 1 and z <= %L;', 10 * counter); + EXECUTE format('explain analyze select *, 1 from t where x <= 1 and y <= %L and z <= 1;', 10 * counter); + EXECUTE format('explain analyze select *, 1 from t where x <= %L and y <= 1 and z <= 1;', 10 * counter); + end loop; +END; +$$ LANGUAGE PLPGSQL; +-- AQO should predict ~1000 rows to indicate that the record from previous invocation was used. +SELECT str AS result +FROM expln(' +select * from t where x <= 10000 and y <= 10000 and z <= 10000;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + result +------------------------------------------------------------------ + Seq Scan on public.t (actual rows=10000 loops=1) + AQO: rows=9987, error=-0% + Output: x, y, z + Filter: ((t.x <= 10000) AND (t.y <= 10000) AND (t.z <= 10000)) + Using aqo: true + AQO mode: LEARN + JOINS: 0 +(7 rows) + RESET aqo.wide_search; +RESET aqo.predict_with_few_neighbors; +RESET aqo.min_neighbors_for_predicting; DROP EXTENSION aqo CASCADE; DROP TABLE a; DROP TABLE b; DROP TABLE c; +DROP TABLE t; DROP FUNCTION expln; diff --git a/expected/unsupported.out b/expected/unsupported.out index 9db07618..a088a47c 100644 --- a/expected/unsupported.out +++ b/expected/unsupported.out @@ -16,6 +16,7 @@ $$ LANGUAGE PLPGSQL; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; DROP TABLE IF EXISTS t; +NOTICE: table "t" does not exist, skipping CREATE TABLE t AS SELECT (gs.* / 50) AS x FROM generate_series(1,1000) AS gs; ANALYZE t; CREATE TABLE t1 AS SELECT mod(gs,10) AS x, mod(gs+1,10) AS y diff --git a/machine_learning.c b/machine_learning.c index bfdf0aaa..d7520a94 100644 --- a/machine_learning.c +++ b/machine_learning.c @@ -150,6 +150,7 @@ OkNNr_predict(OkNNrdata *data, double *features) if (!aqo_predict_with_few_neighbors && data->rows < aqo_k) return -1.; + Assert(data->rows > 0); for (i = 0; i < data->rows; ++i) distances[i] = fs_distance(data->matrix[i], features, data->cols); diff --git a/postprocessing.c b/postprocessing.c index b902e064..cae61e98 100644 --- a/postprocessing.c +++ b/postprocessing.c @@ -95,7 +95,7 @@ atomic_fss_learn_step(uint64 fs, int fss, OkNNrdata *data, double *features, double target, double rfactor, List *reloids) { - if (!load_fss_ext(fs, fss, data, NULL)) + if (!load_aqo_data(fs, fss, data, false)) data->rows = 0; data->rows = OkNNr_learn(data, features, target, rfactor); diff --git a/sql/gucs.sql b/sql/gucs.sql index 0e948cf1..81e245b7 100644 --- a/sql/gucs.sql +++ b/sql/gucs.sql @@ -51,4 +51,5 @@ SELECT count(*) FROM aqo_query_stat; SELECT true AS success FROM aqo_reset(); SELECT count(*) FROM aqo_query_stat; +DROP TABLE t; DROP EXTENSION aqo; diff --git a/sql/look_a_like.sql b/sql/look_a_like.sql index f50e4e55..5eb47a65 100644 --- a/sql/look_a_like.sql +++ b/sql/look_a_like.sql @@ -6,8 +6,9 @@ SET aqo.wide_search = 'on'; SET aqo.mode = 'learn'; SET aqo.show_details = 'on'; -set aqo.show_hash = 'off'; +SET aqo.show_hash = 'off'; SET aqo.min_neighbors_for_predicting = 1; +SET aqo.predict_with_few_neighbors = 'off'; SET enable_nestloop = 'off'; SET enable_mergejoin = 'off'; SET enable_material = 'off'; @@ -142,10 +143,73 @@ FROM expln(' SELECT * FROM (A LEFT JOIN B ON A.x1 = B.y1) sc left join C on sc.x1=C.z1;') AS str WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + +-- Next few test cases focus on fss corresponding to (x1 > ? AND x2 < ? AND x3 < ?). We will denote +-- it by fss0. At this moment there is exactly one fs with (fs, fss0, dbid) record in aqo_data. We'll +-- refer to it as fs0. + +-- Let's create another fs for fss0. We'll call this fs fs1. Since aqo.wide_search='on', +-- aqo.min_neighbors_for_predicting=1, and there is (fs0, fss0, dbid) data record, AQO must be used here. +SELECT str AS result +FROM expln(' +SELECT * FROM A WHERE x1 > -100 AND x2 < 10 AND x3 < 10;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; +-- Now there are 2 data records for fss0: one for (fs0, fss0, dbid) and one for (fs1, fss0, dbid) + +-- We repeat previous query, but set aqo.min_neighbors_for_predicting to 2. Since aqo.predict_with_few_neighbors +-- is 'off', AQO is obliged to use both data records for fss0. +SET aqo.min_neighbors_for_predicting = 2; +SELECT str AS result +FROM expln(' +SELECT * FROM A WHERE x1 > 1 AND x2 < 10 AND x3 < 10;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; +-- Now there are 3 data records for fss0: 1 for (fs0, fss0, dbid) and 2 for (fs1, fss0, dbid) + +-- Lastly, we run invoke query with previously unseen fs with fss0 feature subspace. AQO must use +-- three data records from two neighbors for this one. +SET aqo.min_neighbors_for_predicting = 3; +SELECT str AS result +FROM expln(' +SELECT x2 FROM A WHERE x1 > 3 AND x2 < 10 AND x3 < 10 GROUP BY(x2);') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + +----- +DROP TABLE IF EXISTS t; +CREATE TABLE t AS SELECT x, x AS y, x AS z FROM generate_series(1, 10000) x; +ANALYZE t; +SELECT true AS success FROM aqo_reset(); + +-- Test that when there are less records than aqo.min_neighbors_for_predicting for given (fs, fss, dbid) +-- and aqo.predict_with_few_neighbors is off, those records have higher precedence for cardinality estimation +-- than neighbors' records. +SELECT str AS result +FROM expln(' +select * from t where x <= 10000 and y <= 10000 and z <= 10000;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; +DO +$$ +BEGIN + for counter in 1..20 loop + EXECUTE format('explain analyze select *, 1 from t where x <= 1 and y <= 1 and z <= %L;', 10 * counter); + EXECUTE format('explain analyze select *, 1 from t where x <= 1 and y <= %L and z <= 1;', 10 * counter); + EXECUTE format('explain analyze select *, 1 from t where x <= %L and y <= 1 and z <= 1;', 10 * counter); + end loop; +END; +$$ LANGUAGE PLPGSQL; +-- AQO should predict ~1000 rows to indicate that the record from previous invocation was used. +SELECT str AS result +FROM expln(' +select * from t where x <= 10000 and y <= 10000 and z <= 10000;') AS str +WHERE str NOT LIKE 'Query Identifier%' and str NOT LIKE '%Memory%' and str NOT LIKE '%Sort Method%'; + + RESET aqo.wide_search; +RESET aqo.predict_with_few_neighbors; +RESET aqo.min_neighbors_for_predicting; DROP EXTENSION aqo CASCADE; DROP TABLE a; DROP TABLE b; DROP TABLE c; +DROP TABLE t; DROP FUNCTION expln; diff --git a/storage.c b/storage.c index 10b7cfc6..79b1b11d 100644 --- a/storage.c +++ b/storage.c @@ -120,12 +120,6 @@ PG_FUNCTION_INFO_V1(aqo_query_stat_update); PG_FUNCTION_INFO_V1(aqo_data_update); -bool -load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids) -{ - return load_aqo_data(fs, fss, data, reloids, false, NULL); -} - bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids) { @@ -1577,66 +1571,53 @@ fs_distance(double *a, double *b, int len) } static bool -nearest_neighbor(double **matrix, int old_rows, double *neibour, int cols) +nearest_neighbor(double **matrix, int old_rows, double *neighbor, int cols) { int i; for (i=0; irows is kept <= aqo_K. + */ static void -build_knn_matrix(OkNNrdata *data, const OkNNrdata *temp_data, double *features) +update_knn_matrix(OkNNrdata *data, const OkNNrdata *temp_data) { + int k = (data->rows < 0) ? 0 : data->rows; + int i; + Assert(data->cols == temp_data->cols); Assert(data->matrix); - if (features != NULL) + if (data->cols > 0) { - int old_rows = data->rows; - int k = (old_rows < 0) ? 0 : old_rows; - - if (data->cols > 0) + for (i = 0; i < temp_data->rows && k < aqo_K; i++) { - int i; - - Assert(data->cols == temp_data->cols); - - for (i = 0; i < temp_data->rows; i++) + if (!nearest_neighbor(data->matrix, k, temp_data->matrix[i], data->cols)) { - if (k < aqo_K && !nearest_neighbor(data->matrix, old_rows, - temp_data->matrix[i], - data->cols)) - { - memcpy(data->matrix[k], temp_data->matrix[i], data->cols * sizeof(double)); - data->rfactors[k] = temp_data->rfactors[i]; - data->targets[k] = temp_data->targets[i]; - k++; - } + memcpy(data->matrix[k], temp_data->matrix[i], data->cols * sizeof(double)); + data->rfactors[k] = temp_data->rfactors[i]; + data->targets[k] = temp_data->targets[i]; + k++; } - data->rows = k; } } - else + /* Data has no columns. Only one record can be added */ + else if (k == 0 && temp_data->rows > 0) { - if (data->rows > 0) - /* trivial strategy - use first suitable record and ignore others */ - return; - memcpy(data, temp_data, sizeof(OkNNrdata)); - if (data->cols > 0) - { - int i; - - for (i = 0; i < data->rows; i++) - { - Assert(data->matrix[i]); - memcpy(data->matrix[i], temp_data->matrix[i], data->cols * sizeof(double)); - } - } + data->rfactors[0] = temp_data->rfactors[0]; + data->targets[0] = temp_data->targets[0]; + k = 1; } + data->rows = k; + + Assert(data->rows >= 0 && data->rows <= aqo_K); } static OkNNrdata * @@ -1706,13 +1687,11 @@ _fill_knn_data(const DataEntry *entry, List **reloids) * * If wideSearch is true - make seqscan on the hash table to see for relevant * data across neighbours. - * If reloids is NULL - don't fill this list. * * Return false if the operation was unsuccessful. */ bool -load_aqo_data(uint64 fs, int fss, OkNNrdata *data, List **reloids, - bool wideSearch, double *features) +load_aqo_data(uint64 fs, int fss, OkNNrdata *data, bool wideSearch) { DataEntry *entry; bool found; @@ -1720,6 +1699,7 @@ load_aqo_data(uint64 fs, int fss, OkNNrdata *data, List **reloids, OkNNrdata *temp_data; Assert(!LWLockHeldByMe(&aqo_state->data_lock)); + Assert(wideSearch || data->rows <= 0); dsa_init(); @@ -1739,16 +1719,16 @@ load_aqo_data(uint64 fs, int fss, OkNNrdata *data, List **reloids, if (entry->cols != data->cols) { /* Collision happened? */ - elog(LOG, "[AQO] Does a collision happened? Check it if possible " + elog(LOG, "[AQO] Did a collision happen? Check it if possible " "(fs: "UINT64_FORMAT", fss: %d).", fs, fss); found = false; /* Sign of unsuccessful operation */ goto end; } - temp_data = _fill_knn_data(entry, reloids); + temp_data = _fill_knn_data(entry, NULL); Assert(temp_data->rows > 0); - build_knn_matrix(data, temp_data, features); + update_knn_matrix(data, temp_data); Assert(data->rows > 0); } else @@ -1770,28 +1750,31 @@ load_aqo_data(uint64 fs, int fss, OkNNrdata *data, List **reloids, temp_data = _fill_knn_data(entry, &tmp_oids); - if (data->rows > 0 && list_length(tmp_oids) != noids) + if (noids >= 0 && list_length(tmp_oids) != noids) { /* Dubious case. So log it and skip these data */ elog(LOG, "[AQO] different number depended oids for the same fss %d: " "%d and %d correspondingly.", fss, list_length(tmp_oids), noids); - Assert(noids >= 0); list_free(tmp_oids); continue; } noids = list_length(tmp_oids); + list_free(tmp_oids); - if (reloids != NULL && *reloids == NIL) - *reloids = tmp_oids; - else - list_free(tmp_oids); - - build_knn_matrix(data, temp_data, NULL); + update_knn_matrix(data, temp_data); found = true; + + /* Abort if data is full */ + if (data->rows == aqo_K || (data->cols == 0 && data->rows == 1)) + { + hash_seq_term(&hash_seq); + break; + } } + } Assert(!found || (data->rows > 0 && data->rows <= aqo_K)); diff --git a/storage.h b/storage.h index 9491e33e..692014c3 100644 --- a/storage.h +++ b/storage.h @@ -144,8 +144,7 @@ extern void aqo_qtexts_load(void); extern bool aqo_data_store(uint64 fs, int fss, AqoDataArgs *data, List *reloids); -extern bool load_aqo_data(uint64 fs, int fss, OkNNrdata *data, List **reloids, - bool wideSearch, double *features); +extern bool load_aqo_data(uint64 fs, int fss, OkNNrdata *data, bool wideSearch); extern void aqo_data_flush(void); extern void aqo_data_load(void); @@ -166,7 +165,6 @@ extern bool query_is_deactivated(uint64 query_hash); extern void add_deactivated_query(uint64 query_hash); /* Storage interaction */ -extern bool load_fss_ext(uint64 fs, int fss, OkNNrdata *data, List **reloids); extern bool update_fss_ext(uint64 fs, int fss, OkNNrdata *data, List *reloids); extern bool update_query_timeout(uint64 queryid, int64 smart_timeout);