Skip to content
This repository was archived by the owner on May 22, 2025. It is now read-only.

Improve logging for load_file #1312

Merged
phanikumv merged 1 commit into from
Nov 29, 2022
Merged

Improve logging for load_file #1312

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions python-sdk/src/astro/databases/base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ class BaseDatabase(ABC):
illegal_column_name_chars: list[str] = []
illegal_column_name_chars_replacement: list[str] = []
NATIVE_PATHS: dict[Any, Any] = {}
NATIVE_LOAD_EXCEPTIONS: Any = DatabaseCustomError
DEFAULT_SCHEMA = SCHEMA
NATIVE_AUTODETECT_SCHEMA_CONFIG: Mapping[FileLocation, Mapping[str, list[FileType] | Callable]] = {}
FILE_PATTERN_BASED_AUTODETECT_SCHEMA_SUPPORTED: set[FileLocation] = set()
Expand Down Expand Up @@ -475,6 +474,7 @@ def load_file_to_table_using_pandas(
if_exists: LoadExistStrategy = "replace",
chunk_size: int = DEFAULT_CHUNK_SIZE,
):
logging.info("Loading file(s) with Pandas...")
input_files = resolve_file_path_pattern(
input_file.path,
input_file.conn_id,
Expand Down Expand Up @@ -514,20 +514,21 @@ def load_file_to_table_natively_with_fallback(
"""

try:
logging.info("Loading file(s) with Native Support...")
self.load_file_to_table_natively(
source_file=source_file,
target_table=target_table,
if_exists=if_exists,
native_support_kwargs=native_support_kwargs,
**kwargs,
)
# Catching NATIVE_LOAD_EXCEPTIONS for fallback
except self.NATIVE_LOAD_EXCEPTIONS as load_exception: # skipcq: PYL-W0703
except DatabaseCustomError:
logging.warning(
"Loading files failed with Native Support. Falling back to Pandas-based load",
"Loading file(s) failed with Native Support.",
exc_info=True,
)
if enable_native_fallback:
logging.warning("Falling back to Pandas-based load...")
self.load_file_to_table_using_pandas(
input_file=source_file,
output_table=target_table,
Expand All @@ -536,7 +537,7 @@ def load_file_to_table_natively_with_fallback(
chunk_size=chunk_size,
)
else:
raise load_exception
raise

def load_pandas_dataframe_to_table(
self,
Expand Down