Skip to content
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
48 commits
Select commit Hold shift + click to select a range
9e2f527
leverage enter and exit magic functions for with statements
hughhhh Mar 3, 2021
6730835
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 3, 2021
80cd5bd
setup new logger pattern
hughhhh Mar 3, 2021
f4c8a5f
Update superset/utils/log.py
hughhhh Mar 4, 2021
e2ab146
update test to incorporate user_id
hughhhh Mar 4, 2021
440a9d6
Merge branch 'hugh/event-logger-refactor' of https://github.com/apach…
hughhhh Mar 4, 2021
1266d12
update test
hughhhh Mar 4, 2021
d7e1247
fix mypy
hughhhh Mar 4, 2021
b130962
Apply suggestions from code review
hughhhh Mar 4, 2021
9718486
update test
hughhhh Mar 4, 2021
2d05a37
add logs to database/create
hughhhh Mar 4, 2021
f9d5f3f
add logs to test_connection
hughhhh Mar 4, 2021
58fcd28
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 4, 2021
47fe377
fix pylint
hughhhh Mar 4, 2021
2fda45e
Merge branch 'hugh/event-logger-refactor' of https://github.com/apach…
hughhhh Mar 4, 2021
35763ef
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 4, 2021
31abdc8
pylint
hughhhh Mar 5, 2021
12e7731
Merge branch 'hugh/event-logger-refactor' of https://github.com/apach…
hughhhh Mar 5, 2021
e0871f0
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 5, 2021
bcf85f8
fix celery test
hughhhh Mar 5, 2021
f42efe2
fix celery test
hughhhh Mar 5, 2021
82612ea
fix all cta test
hughhhh Mar 5, 2021
9d7fc0e
add exception block for no user
hughhhh Mar 5, 2021
bfa5477
fixed test
hughhhh Mar 5, 2021
68f25d0
reup celery test
hughhhh Mar 5, 2021
f483764
pylint
hughhhh Mar 5, 2021
538d9dd
Merge branch 'hugh/event-logger-refactor' of https://github.com/apach…
hughhhh Mar 5, 2021
ea81349
event logger
hughhhh Mar 5, 2021
46ce47f
add pyline
hughhhh Mar 5, 2021
f4eaac1
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 5, 2021
8f6d379
Merge branch 'master' of https://github.com/apache/incubator-superset…
hughhhh Mar 6, 2021
cfbab0c
fix test
hughhhh Mar 7, 2021
55601ee
added 1 more test
hughhhh Mar 7, 2021
cffe48e
add more test
hughhhh Mar 7, 2021
c4039f9
add name
hughhhh Mar 7, 2021
b2c120c
address comment
hughhhh Mar 8, 2021
2f86e2a
fix docstring
hughhhh Mar 8, 2021
3856999
updated test
hughhhh Mar 8, 2021
33e4180
update security test
hughhhh Mar 8, 2021
67d42b4
update test
hughhhh Mar 8, 2021
f11ab8a
fix annotation on DatabaseDAO.build_db_for_connection_test
hughhhh Mar 8, 2021
7c28363
fix
hughhhh Mar 9, 2021
ce0240a
make the names consistent
hughhhh Mar 9, 2021
e20f6dd
remove unused packages
hughhhh Mar 9, 2021
864173d
fix test
hughhhh Mar 9, 2021
a904cd8
fix test
hughhhh Mar 9, 2021
46095ca
fix connection test
hughhhh Mar 9, 2021
c48ecd7
fix connection test
hughhhh Mar 9, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions superset/databases/commands/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
)
from superset.databases.commands.test_connection import TestConnectionDatabaseCommand
from superset.databases.dao import DatabaseDAO
from superset.extensions import db, security_manager
from superset.extensions import db, event_logger, security_manager

logger = logging.getLogger(__name__)

Expand All @@ -50,9 +50,13 @@ def run(self) -> Model:

try:
TestConnectionDatabaseCommand(self._actor, self._properties).run()
except Exception:
db.session.rollback()
raise DatabaseConnectionFailedError()
except Exception: # pylint: disable=broad-except
with event_logger(
action="db_connection_failed",
engine=database.db_engine_spec.__name__,
):
db.session.rollback()
raise DatabaseConnectionFailedError()

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For places where we do a rollback we should it outside the context manager, otherwise if the logging fails the transaction won't be rolled back:

Suggested change
with event_logger(
action="db_connection_failed",
engine=database.db_engine_spec.__name__,
):
db.session.rollback()
raise DatabaseConnectionFailedError()
db.session.rollback()
event_logger.log_with_context(
action="db_connection_failed",
engine=database.db_engine_spec.__name__,
)
raise DatabaseConnectionFailedError()

Reversing the order ensures that even if the logging fails the rollback will still go through.


# adding a new database we always want to force refresh schema list
schemas = database.get_all_schema_names(cache=False)
Expand All @@ -63,8 +67,11 @@ def run(self) -> Model:
security_manager.add_permission_view_menu("database_access", database.perm)
db.session.commit()
except DAOCreateFailedError as ex:
logger.exception(ex.exception)
raise DatabaseCreateFailedError()
with event_logger(
action=f"db_creation_failed.{ex.exception}",
Comment thread
betodealmeida marked this conversation as resolved.
Outdated
engine=database.db_engine_spec.__name__,

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@hughhhh what did you want to do about making the engine string consistent around each of these logs so that we can group by the same engine later when observing the logs?

@hughhhh hughhhh Mar 9, 2021

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I ended up changing all the engine references in the event_logger.log_with_context to pull data from the database.db_engine_spec.__name__ I say we go with this for now and see the data then see if we need to create some mapping either in the service or downstream

):
raise DatabaseCreateFailedError()
return database

def validate(self) -> None:
Expand All @@ -84,4 +91,5 @@ def validate(self) -> None:
if exceptions:
exception = DatabaseInvalidError()
exception.add_list(exceptions)
raise exception
with event_logger(action="db_connection_failed"):
raise exception
53 changes: 39 additions & 14 deletions superset/databases/commands/test_connection.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
)
from superset.databases.dao import DatabaseDAO
from superset.exceptions import SupersetSecurityException
from superset.extensions import event_logger
from superset.models.core import Database

logger = logging.getLogger(__name__)
Expand All @@ -55,25 +56,49 @@ def run(self) -> None:
impersonate_user=self._properties.get("impersonate_user", False),
encrypted_extra=self._properties.get("encrypted_extra", "{}"),
)
if database is not None:
database.set_sqlalchemy_uri(uri)
database.db_engine_spec.mutate_db_for_connection_test(database)
username = self._actor.username if self._actor is not None else None
engine = database.get_sqla_engine(user_name=username)
if database is None:
raise DBAPIError("Database is not found", None, None)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can actually remove this and fix the type annotation in DatabaseDAO.build_db_for_connection_test, the class method never returns None.


database.set_sqlalchemy_uri(uri)
database.db_engine_spec.mutate_db_for_connection_test(database)
username = self._actor.username if self._actor is not None else None
engine = database.get_sqla_engine(user_name=username)
with closing(engine.raw_connection()) as conn:
if not engine.dialect.do_ping(conn):
raise DBAPIError(None, None, None)
except (NoSuchModuleError, ModuleNotFoundError):

with event_logger(
action="test_connection_success", engine=make_url(uri).drivername
):
pass

except (NoSuchModuleError, ModuleNotFoundError) as ex:
driver_name = make_url(uri).drivername
raise DatabaseTestConnectionDriverError(
message=_("Could not load database driver: {}").format(driver_name),
)
except DBAPIError:
raise DatabaseTestConnectionFailedError()
with event_logger(
action=f"test_connection_error.{ex.__class__.__name__}",
engine=make_url(uri).drivername,
):
raise DatabaseTestConnectionDriverError(
message=_("Could not load database driver: {}").format(driver_name),
)
except DBAPIError as ex:
with event_logger(
action=f"test_connection_error.{ex.__class__.__name__}",
engine=make_url(uri).drivername,
):
raise DatabaseTestConnectionFailedError()
except SupersetSecurityException as ex:
raise DatabaseSecurityUnsafeError(message=str(ex))
except Exception:
raise DatabaseTestConnectionUnexpectedError()
with event_logger(
action=f"test_connection_error.{ex.__class__.__name__}",
engine=make_url(uri).drivername,
):
raise DatabaseSecurityUnsafeError(message=str(ex))
except Exception as ex: # pylint: disable=broad-except
with event_logger(
action=f"test_connection_error.{ex.__class__.__name__}",
engine=make_url(uri).drivername,
):
raise DatabaseTestConnectionUnexpectedError()

def validate(self) -> None:
database_name = self._properties.get("database_name")
Expand Down