From d694871bbd6b0b883790936fb0e0b8834e01ab9b Mon Sep 17 00:00:00 2001 From: Remi Dettai Date: Mon, 20 Jan 2025 10:27:38 +0100 Subject: [PATCH] Clarify tag_fields types --- docs/configuration/index-config.md | 2 +- ..._search.yaml => 0002_negative_search.yaml} | 0 ...ts_search.yaml => 0003_exists_search.yaml} | 0 .../qw_search_api/_setup.quickwit.yaml | 40 ------ .../qw_search_api/_teardown.quickwit.yaml | 4 - .../tag_fields/0001_allowed_types.yaml | 133 ++++++++++++++++++ .../0002_negative_tags.yaml | 8 +- .../scenarii/tag_fields/_ctx.yaml | 5 + .../scenarii/tag_fields/_setup.quickwit.yaml | 45 ++++++ .../tag_fields/_teardown.quickwit.yaml | 6 + 10 files changed, 194 insertions(+), 49 deletions(-) rename quickwit/rest-api-tests/scenarii/qw_search_api/{0003_negative_search.yaml => 0002_negative_search.yaml} (100%) rename quickwit/rest-api-tests/scenarii/qw_search_api/{0004_exists_search.yaml => 0003_exists_search.yaml} (100%) create mode 100644 quickwit/rest-api-tests/scenarii/tag_fields/0001_allowed_types.yaml rename quickwit/rest-api-tests/scenarii/{qw_search_api => tag_fields}/0002_negative_tags.yaml (74%) create mode 100644 quickwit/rest-api-tests/scenarii/tag_fields/_ctx.yaml create mode 100644 quickwit/rest-api-tests/scenarii/tag_fields/_setup.quickwit.yaml create mode 100644 quickwit/rest-api-tests/scenarii/tag_fields/_teardown.quickwit.yaml diff --git a/docs/configuration/index-config.md b/docs/configuration/index-config.md index e7376aaf4bc..d7ffbb6094f 100644 --- a/docs/configuration/index-config.md +++ b/docs/configuration/index-config.md @@ -91,7 +91,7 @@ The doc mapping defines how a document and the fields it contains are stored and | `field_mappings` | Collection of field mapping, each having its own data type (text, binary, datetime, bool, i64, u64, f64, ip, json). | `[]` | | `mode` | Defines how quickwit should handle document fields that are not present in the `field_mappings`. In particular, the "dynamic" mode makes it possible to use quickwit in a schemaless manner. (See [mode](#mode)) | `dynamic` | `dynamic_mapping` | This parameter is only allowed when `mode` is set to `dynamic`. It then defines whether dynamically mapped fields should be indexed, stored, etc. | (See [mode](#mode)) -| `tag_fields` | Collection of fields* already defined in `field_mappings` whose values will be stored as part of the `tags` metadata. [Learn more about tags](../overview/concepts/querying.md#tag-pruning). | `[]` | +| `tag_fields` | Collection of fields* explicitly defined in `field_mappings` whose values will be stored as part of the `tags` metadata. Allowed types are: `text` (with raw tokenizer), `i64` and `u64`. [Learn more about tags](../overview/concepts/querying.md#tag-pruning). | `[]` | | `store_source` | Whether or not the original JSON document is stored or not in the index. | `false` | | `timestamp_field` | Timestamp field* used for sharding documents in splits. The field has to be of type `datetime`. [Learn more about time sharding](./../overview/architecture.md). | `None` | | `partition_key` | If set, quickwit will route documents into different splits depending on the field name declared as the `partition_key`. | `null` | diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0003_negative_search.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/0002_negative_search.yaml similarity index 100% rename from quickwit/rest-api-tests/scenarii/qw_search_api/0003_negative_search.yaml rename to quickwit/rest-api-tests/scenarii/qw_search_api/0002_negative_search.yaml diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0004_exists_search.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/0003_exists_search.yaml similarity index 100% rename from quickwit/rest-api-tests/scenarii/qw_search_api/0004_exists_search.yaml rename to quickwit/rest-api-tests/scenarii/qw_search_api/0003_exists_search.yaml diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml index e5865e2c2c1..5ccac87b044 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/_setup.quickwit.yaml @@ -45,46 +45,6 @@ ndjson: - {"not_fast": 1684993003} --- method: DELETE -endpoint: indexes/tagged -status_code: null ---- -method: POST -endpoint: indexes/ -json: - version: "0.7" - index_id: tagged - doc_mapping: - field_mappings: - - name: seq - type: u64 - - name: tag - type: u64 - tag_fields: ["tag"] ---- -method: POST -endpoint: tagged/ingest -params: - commit: force -ndjson: - - {"seq": 1, "tag": 1} - - {"seq": 2, "tag": 2} ---- -method: POST -endpoint: tagged/ingest -params: - commit: force -ndjson: - - {"seq": 1, "tag": 1} - - {"seq": 3, "tag": null} ---- -method: POST -endpoint: tagged/ingest -params: - commit: force -ndjson: - - {"seq": 4, "tag": 1} ---- -method: DELETE endpoint: indexes/nested status_code: null --- diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/qw_search_api/_teardown.quickwit.yaml index 36dd9c24422..56cd2bda8a9 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/_teardown.quickwit.yaml +++ b/quickwit/rest-api-tests/scenarii/qw_search_api/_teardown.quickwit.yaml @@ -1,9 +1,5 @@ -# Delete index method: DELETE endpoint: indexes/simple --- method: DELETE -endpoint: indexes/tagged ---- -method: DELETE endpoint: indexes/nested diff --git a/quickwit/rest-api-tests/scenarii/tag_fields/0001_allowed_types.yaml b/quickwit/rest-api-tests/scenarii/tag_fields/0001_allowed_types.yaml new file mode 100644 index 00000000000..648306a84ac --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/tag_fields/0001_allowed_types.yaml @@ -0,0 +1,133 @@ +# allowed types +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: allowedtypes + doc_mapping: + mode: dynamic + field_mappings: + - name: text1 + type: text + tokenizer: raw + - name: number1 + type: u64 + - name: number2 + type: i64 + tag_fields: + - text1 + - number1 + - number2 +--- +# tokenized not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: tokenizedtype + doc_mapping: + mode: dynamic + field_mappings: + - name: text1 + type: text + tokenizer: default + tag_fields: [text1] +status_code: 400 +--- +# float not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: floattype + doc_mapping: + mode: dynamic + field_mappings: + - name: number3 + type: f64 + tag_fields: [number3] +status_code: 400 +--- +# boolean not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: booltype + doc_mapping: + mode: dynamic + field_mappings: + - name: boolean + type: bool + tag_fields: [boolean] +status_code: 400 +--- +# json not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: jsontype + doc_mapping: + mode: dynamic + field_mappings: + - name: json1 + type: json + tag_fields: [json1] +status_code: 400 +--- +# ip not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: iptype + doc_mapping: + mode: dynamic + field_mappings: + - name: ip1 + type: ip + tag_fields: [ip1] +status_code: 400 +--- +# bytes not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: bytestype + doc_mapping: + mode: dynamic + field_mappings: + - name: bytes1 + type: bytes + tag_fields: [bytes1] +status_code: 400 +--- +# bytes not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: datetype + doc_mapping: + mode: dynamic + field_mappings: + - name: date1 + type: datetime + input_formats: + - rfc3339 + tag_fields: [date1] +status_code: 400 +--- +# dynamic not allowed +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: dynamictype + doc_mapping: + mode: dynamic + tag_fields: [dynamic1] +status_code: 400 +--- diff --git a/quickwit/rest-api-tests/scenarii/qw_search_api/0002_negative_tags.yaml b/quickwit/rest-api-tests/scenarii/tag_fields/0002_negative_tags.yaml similarity index 74% rename from quickwit/rest-api-tests/scenarii/qw_search_api/0002_negative_tags.yaml rename to quickwit/rest-api-tests/scenarii/tag_fields/0002_negative_tags.yaml index 0b716eb9a1b..f1f900c342c 100644 --- a/quickwit/rest-api-tests/scenarii/qw_search_api/0002_negative_tags.yaml +++ b/quickwit/rest-api-tests/scenarii/tag_fields/0002_negative_tags.yaml @@ -1,23 +1,23 @@ # regression test for https://github.com/quickwit-oss/quickwit/issues/4698 -endpoint: tagged/search +endpoint: simple/search params: query: "tag:1" expected: num_hits: 3 --- -endpoint: tagged/search +endpoint: simple/search params: query: "-tag:2" expected: num_hits: 4 --- -endpoint: tagged/search +endpoint: simple/search params: query: "tag:2" expected: num_hits: 1 --- -endpoint: tagged/search +endpoint: simple/search params: query: "-tag:1" expected: diff --git a/quickwit/rest-api-tests/scenarii/tag_fields/_ctx.yaml b/quickwit/rest-api-tests/scenarii/tag_fields/_ctx.yaml new file mode 100644 index 00000000000..f507346eae6 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/tag_fields/_ctx.yaml @@ -0,0 +1,5 @@ +method: GET +engines: ["quickwit"] +api_root: "http://localhost:7280/api/v1/" +headers: + Content-Type: application/json diff --git a/quickwit/rest-api-tests/scenarii/tag_fields/_setup.quickwit.yaml b/quickwit/rest-api-tests/scenarii/tag_fields/_setup.quickwit.yaml new file mode 100644 index 00000000000..1208ca48343 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/tag_fields/_setup.quickwit.yaml @@ -0,0 +1,45 @@ +# Delete possibly remaining index +method: DELETE +endpoint: indexes/allowedtypes +status_code: null +--- +method: DELETE +endpoint: indexes/simple +status_code: null +--- +method: POST +endpoint: indexes/ +json: + version: "0.7" + index_id: simple + doc_mapping: + field_mappings: + - name: seq + type: u64 + - name: tag + type: u64 + tag_fields: ["tag"] +--- +method: POST +endpoint: simple/ingest +params: + commit: force +ndjson: + - {"seq": 1, "tag": 1} + - {"seq": 2, "tag": 2} +--- +method: POST +endpoint: simple/ingest +params: + commit: force +ndjson: + - {"seq": 1, "tag": 1} + - {"seq": 3, "tag": null} +--- +method: POST +endpoint: simple/ingest +params: + commit: force +ndjson: + - {"seq": 4, "tag": 1} +--- diff --git a/quickwit/rest-api-tests/scenarii/tag_fields/_teardown.quickwit.yaml b/quickwit/rest-api-tests/scenarii/tag_fields/_teardown.quickwit.yaml new file mode 100644 index 00000000000..fa0ca391b51 --- /dev/null +++ b/quickwit/rest-api-tests/scenarii/tag_fields/_teardown.quickwit.yaml @@ -0,0 +1,6 @@ +method: DELETE +endpoint: indexes/allowedtypes +status_code: null +--- +method: DELETE +endpoint: indexes/simple