Skip to content

Commit

Permalink
Revert "Extended 'max_line_length' when reading in csv file in DuckDB (
Browse files Browse the repository at this point in the history
…#293)"

This reverts commit d884867.
  • Loading branch information
cjohns-scottlogic authored Dec 4, 2024
1 parent efa94fe commit 1d6ece8
Showing 1 changed file with 2 additions and 2 deletions.
4 changes: 2 additions & 2 deletions digital_land/package/datasetparquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,7 @@ def load_entities(self):
SELECT {fields_str} FROM (
SELECT {fields_str}, CASE WHEN resource_csv."end-date" IS NULL THEN '2999-12-31' ELSE resource_csv."end-date" END AS resource_end_date
FROM temp_table
LEFT JOIN read_csv_auto('{self.resource_path}', max_line_size=40000000) resource_csv
LEFT JOIN read_csv_auto('{self.resource_path}') resource_csv
ON temp_table.resource = resource_csv.resource
QUALIFY ROW_NUMBER() OVER (
PARTITION BY entity, field
Expand Down Expand Up @@ -232,7 +232,7 @@ def load_entities(self):
# define organisation query
org_csv = organisation_path
org_query = f"""
SELECT * FROM read_csv_auto('{org_csv}', max_line_size=40000000)
SELECT * FROM read_csv_auto('{org_csv}')
"""

sql = f"""
Expand Down

0 comments on commit 1d6ece8

Please sign in to comment.