Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Added incremental increase to max_line_size" #300

Merged
merged 1 commit into from
Dec 4, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 13 additions & 30 deletions digital_land/package/datasetparquet.py
Original file line number Diff line number Diff line change
Expand Up @@ -65,36 +65,19 @@ def create_temp_table(self, input_paths):
input_paths_str = ", ".join([f"'{path}'" for path in input_paths])

self.conn.execute("DROP TABLE IF EXISTS temp_table")
# Initial max_line_size and increment step
max_size = 40000000
increment_step = 20000000
max_limit = 200000000 # Maximum allowable line size to attempt

while True:
try:
query = f"""
CREATE TEMPORARY TABLE temp_table AS
SELECT *
FROM read_csv(
[{input_paths_str}],
columns = {self.schema},
header = true,
force_not_null = {[field for field in self.schema.keys()]},
max_line_size={max_size}
)
"""
self.conn.execute(query)
break
except duckdb.Error as e: # Catch specific DuckDB error
if "Value with unterminated quote" in str(e):
max_size += increment_step
if max_size > max_limit:
print(
f"Exceeded max_limit of {max_limit}. Could not resolve the issue."
)
raise
else:
raise
query = f"""
CREATE TEMPORARY TABLE temp_table AS
SELECT *
FROM read_csv(
[{input_paths_str}],
columns = {self.schema},
header = true,
force_not_null = {[field for field in self.schema.keys()]},
max_line_size=40000000
)
"""

self.conn.execute(query)

def load_facts(self):
logging.info("loading facts from temp table")
Expand Down
Loading