Merge branch 'dev'

.pre-commit-config.yaml

@@ -5,20 +5,20 @@ repos:
   #     - id: mypy
 
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
+    rev: v5.0.0
     hooks:
       - id: check-yaml
       - id: check-added-large-files
 
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.17.0
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
         args: [--py39-plus, --keep-runtime-typing]
 
   - repo: https://github.com/charliermarsh/ruff-pre-commit
     # Ruff version.
-    rev: v0.5.5
+    rev: v0.7.2
     hooks:
       # Run the linter.
       - id: ruff

cs_tools/__project__.py

@@ -1,4 +1,4 @@
-__version__ = "1.5.10"
+__version__ = "1.5.11"
 __docs__ = "https://thoughtspot.github.io/cs_tools/"
 __repo__ = "https://github.com/thoughtspot/cs_tools"
 __help__ = f"{__repo__}/discussions/"

cs_tools/api/_rest_api_v2.py

@@ -77,6 +77,15 @@ def logs_fetch(
         r = self.request("POST", "/api/rest/2.0/logs/fetch", json=body)
         return r
 
+    # ==================================================================================================================
+    # METADATA ::  https://developers.thoughtspot.com/docs/rest-apiv2-reference#_users
+    # ==================================================================================================================
+
+    def users_delete(self, user_identifier: Identifier) -> httpx.Response:
+        body = {"user_identifier": user_identifier}
+        r = self.request("POST", f"/api/rest/2.0/users/{user_identifier}/delete", json=body)
+        return r
+
     # ==================================================================================================================
     # METADATA ::  https://developers.thoughtspot.com/docs/rest-apiv2-reference#_tags
     # ==================================================================================================================

cs_tools/api/middlewares/tsload.py

@@ -342,10 +342,13 @@ def status(self, cycle_id: CycleID, *, wait_for_complete: bool = False) -> dict[
             if not wait_for_complete:
                 break
 
+            if data["internal_stage"] in ("COMMITTING", "INGESTING"):
+                continue
+
             if data["internal_stage"] == "DONE":
                 break
 
-            if data["status"]["message"] != "OK":
+            if "code" in data["status"] and data["status"]["code"] != "OK":
                 break
 
             log.debug(f"data load status:\n{data}")

cs_tools/cli/commands/self.py

@@ -80,7 +80,7 @@ def update(
             raise typer.Exit(0)
 
     log.info("Upgrading CS Tools and its dependencies.")
-    cs_tools_venv.pip("install", *requires, "--upgrade", "--upgrade-strategy", "eager")
+    cs_tools_venv.pip("install", *requires, "--upgrade", "--upgrade-strategy", "eager", raise_on_failure=False)
 
 
 @app.command(cls=CSToolsCommand)

cs_tools/cli/keyboard/keys.py

@@ -53,7 +53,9 @@ def _cache_singleton_representation(self) -> Key:
         # Cache everything except paste key events, because the data will always be different.
         if self._is_pasted_characters:
             return self
-        return _known_keys_cache.setdefault(self.name, self)
+
+        _known_keys_cache.setdefault(self.name, self)
+        return self
 
     @classmethod
     def letter(cls, value: str) -> Key:

cs_tools/cli/tools/scriptability/_export.py

@@ -148,6 +148,11 @@ def export(
                     )
                 )
 
+            except Exception as e:
+                log.error(f"Something went wrong went extracting {guid}, see logs for more details..")
+                log.debug(f"Full error: {e}", exc_info=True)
+                continue
+
     if results:
         _show_results_as_table(results=results)
     else:

cs_tools/cli/tools/scriptability/tmlfs.py

@@ -2,6 +2,7 @@
 This file contains code for the TML file system.  The TML file system is a structured file system for storing TML and
 related files.
 """
+
 from __future__ import annotations
 
 import datetime as dt
@@ -33,6 +34,7 @@ class TMLType(StrEnum):
     sql_view = "sql_view"
     sqlview = "sqlview"
     worksheet = "worksheet"
+    model = "model"
     answer = "answer"
     liveboard = "liveboard"
     pinboard = "pinboard"
@@ -111,7 +113,9 @@ def _create_log_path(self, log_for: str) -> pathlib.Path:
 
         # make sure the path already exists.
         if not self.path.exists():
-            raise CSToolsCLIError(title=f"Path {self.path} does not exist.", mitigation="Specify a valid path to the FS.")
+            raise CSToolsCLIError(
+                title=f"Path {self.path} does not exist.", mitigation="Specify a valid path to the FS."
+            )
 
         # create the log folder and file.
         log_name = f"{log_for}-{self.start_time.strftime('%Y.%m.%d-%H.%M.%S')}"

cs_tools/cli/tools/searchable/app.py

@@ -17,7 +17,7 @@
 from cs_tools.cli.layout import LiveTasks
 from cs_tools.cli.types import SyncerProtocolType, TZAwareDateTimeType
 from cs_tools.cli.ux import CSToolsApp, rich_console
-from cs_tools.sync.csv.syncer import CSV
+from cs_tools.sync.sqlite.syncer import SQLite
 from cs_tools.types import GUID, TMLImportPolicy
 
 from . import layout, models, transform
@@ -162,7 +162,7 @@ def deploy(
                 logger(f"{divider} [b blue]{r.name}[/] {divider} {info}")
 
 
-@app.command(dependencies=[thoughtspot], hidden=True)
+@app.command(dependencies=[thoughtspot])
 def audit_logs(
     ctx: typer.Context,
     syncer: DSyncer = typer.Option(
@@ -429,11 +429,12 @@ def metadata(
         log.warning("Searchable is meant to be run from an Admin-level context, your results may vary..")
 
     table = layout.Table(data=[[str(org)] + [":popcorn:"] * 8 for org in orgs])
-    temp_sync = CSV(directory=ts.config.temp_dir, empty_as_null=True, save_strategy="APPEND")
+    temp_sync = SQLite(
+        database_path=ts.config.temp_dir / "temp-data.db", models=models.METADATA_MODELS, load_strategy="UPSERT"
+    )
 
-    # Orgs have the potential for having limited data, let's be less noisy.
-    logger = logging.getLogger("cs_tools.sync.csv.syncer")
-    logger.setLevel("ERROR")
+    # Silence the intermediate logger.
+    logging.getLogger("cs_tools.sync.sqlite.syncer").setLevel(logging.CRITICAL)
 
     with Live(table, console=rich_console, auto_refresh=1):
         temp_sync.dump(models.Cluster.__tablename__, data=transform.to_cluster(ts.session_context))
@@ -505,8 +506,7 @@ def metadata(
                 for m in temp_sync.load(models.MetadataObject.__tablename__)
             }
             temp_sync.dump(
-                models.DataSource.__tablename__,
-                data=transform.to_data_source(content, cluster=cluster_uuid),
+                models.DataSource.__tablename__, data=transform.to_data_source(content, cluster=cluster_uuid)
             )
             temp_sync.dump(
                 models.MetadataObject.__tablename__,
@@ -561,14 +561,11 @@ def metadata(
 
             # GO TO NEXT ORG BATCH -->
 
-        # RESTORE CSV logger.level
-        logger.setLevel("DEBUG")
-
         # WRITE ALL THE COMBINED DATA TO THE TARGET SYNCER
         is_syncer_initialized_as_db_truncate = syncer.is_database_syncer and syncer.load_strategy == "TRUNCATE"
 
         for model in models.METADATA_MODELS:
-            for idx, rows in enumerate(temp_sync.read_stream(filename=model.__tablename__, batch=1_000_000), start=1):
+            for idx, rows in enumerate(temp_sync.read_stream(tablename=model.__tablename__, batch=1_000_000), start=1):
                 if is_syncer_initialized_as_db_truncate:
                     syncer.load_strategy = "TRUNCATE" if idx == 1 else "APPEND"
 

cs_tools/cli/tools/searchable/models.py

@@ -5,7 +5,7 @@
 import logging
 
 from sqlalchemy.schema import Column
-from sqlalchemy.types import BigInteger, Text
+from sqlalchemy.types import TIMESTAMP, BigInteger, Text
 from sqlmodel import Field
 import pydantic
 
@@ -43,8 +43,8 @@ class User(ValidatedSQLModel, table=True):
     email: Optional[str]
     display_name: str
     sharing_visibility: str
-    created: dt.datetime
-    modified: dt.datetime
+    created: dt.datetime = Field(sa_column=Column(TIMESTAMP))
+    modified: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     user_type: str
 
     @pydantic.field_validator("created", "modified", mode="before")
@@ -62,8 +62,8 @@ class Group(ValidatedSQLModel, table=True):
     description: Optional[str] = Field(sa_column=Column(Text, info={"length_override": "MAX"}))
     display_name: str
     sharing_visibility: str
-    created: dt.datetime
-    modified: dt.datetime
+    created: dt.datetime = Field(sa_column=Column(TIMESTAMP))
+    modified: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     group_type: str
 
     @pydantic.field_validator("description", mode="before")
@@ -105,8 +105,8 @@ class Tag(ValidatedSQLModel, table=True):
     tag_guid: str = Field(primary_key=True)
     tag_name: str
     author_guid: str
-    created: dt.datetime
-    modified: dt.datetime
+    created: dt.datetime = Field(sa_column=Column(TIMESTAMP))
+    modified: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     color: Optional[str]
 
     @pydantic.field_validator("created", "modified", mode="before")
@@ -143,8 +143,8 @@ class MetadataObject(ValidatedSQLModel, table=True):
     name: str = Field(sa_column=Column(Text, info={"length_override": "MAX"}))
     description: Optional[str] = Field(sa_column=Column(Text, info={"length_override": "MAX"}))
     author_guid: str
-    created: dt.datetime
-    modified: dt.datetime
+    created: dt.datetime = Field(sa_column=Column(TIMESTAMP))
+    modified: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     object_type: str
     object_subtype: Optional[str]
     data_source_guid: Optional[str]
@@ -236,8 +236,8 @@ class DependentObject(ValidatedSQLModel, table=True):
     name: str = Field(sa_column=Column(Text, info={"length_override": "MAX"}))
     description: Optional[str] = Field(sa_column=Column(Text, info={"length_override": "MAX"}))
     author_guid: str
-    created: dt.datetime
-    modified: dt.datetime
+    created: dt.datetime = Field(sa_column=Column(TIMESTAMP))
+    modified: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     object_type: str
     object_subtype: Optional[str]
     is_verified: Optional[bool]
@@ -273,7 +273,7 @@ class AuditLogs(ValidatedSQLModel, table=True):
     cluster_guid: str = Field(primary_key=True)
     org_id: int = Field(0, primary_key=True)
     sk_dummy: str = Field(primary_key=True)
-    timestamp: dt.datetime
+    timestamp: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     log_type: str
     user_guid: Optional[str]
     description: str
@@ -286,7 +286,7 @@ class BIServer(ValidatedSQLModel, table=True):
     sk_dummy: str = Field(primary_key=True)
     org_id: int = 0
     incident_id: str
-    timestamp: dt.datetime
+    timestamp: dt.datetime = Field(sa_column=Column(TIMESTAMP))
     url: Optional[str]
     http_response_code: Optional[int]
     browser_type: Optional[str]

cs_tools/cli/tools/searchable/static/CS Tools ~ Worksheet Column Utilization.liveboard.tml

@@ -204,7 +204,7 @@ liveboard:
       formulas:
       - id: Column Type (incl. DATE)
         name: Column Type (incl. DATE)
-        expr: "if ( [Data Type] in { 'date' , 'datetime' , 'timestamp' } ) then sql_string_op ( 'upper({0})' , 'date' ) else [Column Type]"
+        expr: "if ( [Data Type] in { 'date' , 'datetime', 'date_time' , 'timestamp' } ) then sql_string_op ( 'upper({0})' , 'date' ) else [Column Type]"
         was_auto_generated: false
       - id: Formulas
         name: Formulas
@@ -425,7 +425,7 @@ liveboard:
         was_auto_generated: false
       - id: Column Type (incl. DATE)
         name: Column Type (incl. DATE)
-        expr: "if ( [Data Type] in { 'date' , 'datetime' , 'timestamp' } ) then sql_string_op ( 'upper({0})' , 'date' ) else [Column Type]"
+        expr: "if ( [Data Type] in { 'date' , 'datetime' , 'date_time' , 'timestamp' } ) then sql_string_op ( 'upper({0})' , 'date' ) else [Column Type]"
         was_auto_generated: false
       search_query: "[Column] count [Column] [Dependent Interactions] [Is Formula (Pretty)] [Is Hidden (Pretty)] sort by [Column] [Answers] [Liveboards] [Is not Used] unique count [Dependent Author] [Column Type (incl. DATE)] [Worksheet].'(sample) retail - apparel' [Suggestions (Pretty)]"
       answer_columns:

cs_tools/cli/tools/searchable/transform.py

@@ -214,9 +214,9 @@ def to_data_source(data: ArbitraryJsonFormat, cluster: str) -> TableRowsFormat:
                 continue
 
             ever_seen.add((model.cluster_guid, str(model.org_id), model.data_source_guid))
-            out.append(model)
+            out.append(model.model_dump())
 
-    return [model.model_dump() for model in out]
+    return out
 
 
 def to_metadata_object(data: ArbitraryJsonFormat, cluster: str, ever_seen: set[tuple[str, ...]]) -> TableRowsFormat:

cs_tools/cli/tools/user-management/app.py

@@ -10,10 +10,10 @@
 from cs_tools.api._utils import SYSTEM_USERS
 from cs_tools.cli.dependencies import thoughtspot
 from cs_tools.cli.dependencies.syncer import DSyncer
-from cs_tools.cli.layout import LiveTasks
+from cs_tools.cli.layout import ConfirmationListener, LiveTasks
 from cs_tools.cli.types import MetadataType, MultipleChoiceType, SyncerProtocolType
 from cs_tools.cli.ux import CSToolsApp, rich_console
-from cs_tools.errors import CSToolsError, CSToolsCLIError
+from cs_tools.errors import CSToolsCLIError
 
 from . import layout, models, work
 
@@ -197,6 +197,63 @@ def rename(
         log.warning(f"[b yellow]Failed to update {len(failed)} Users\n" + "\n - ".join(failed))
 
 
+@app.command(dependencies=[thoughtspot])
+def delete(
+    ctx: typer.Context,
+    syncer: DSyncer = typer.Option(
+        ...,
+        click_type=SyncerProtocolType(models=models.USER_MODELS),
+        help="protocol and path for options to pass to the syncer",
+        rich_help_panel="Syncer Options",
+    ),
+    deletion: str = typer.Option(..., help="directive to find usernames to sync at", rich_help_panel="Syncer Options"),
+):
+    """
+    Removes Users from ThoughtSpot.
+
+    Your syncer deletion directive (table name or filename, etc..) must follow at least the tabular format below.
+
+    \b
+        +----------------+
+        | username       |
+        +----------------+
+        | cs_tools       |
+        | namey.namerson |
+        | fake.user      |
+        +----------------+
+    """
+    ts = ctx.obj.thoughtspot
+
+    data = syncer.load(deletion)
+
+    if not data:
+        log.warning(f"No users found in {syncer.name} directive '{deletion}'")
+        raise typer.Exit(-1)
+
+    if not all(_.get("username") or _.get("user_guid") for _ in data):
+        log.warning(f"Users in {syncer.name} directive '{deletion}' must have an identifier (username or user_guid)")
+        raise typer.Exit(-1)
+
+    log.warning(f"Would you like to delete {len(data)} users? [b green]Y[/]es/[b red]N[/]o")
+    kb = ConfirmationListener(timeout=60)
+    kb.run()
+
+    if not kb.response.upper() == "Y":
+        log.info("Confirmation [b red]Denied[/] (no users will be deleted)")
+    else:
+        log.info("Confirmation [b green]Approved[/]")
+
+        for row in data:
+            user_identifier = row.get("username") or row.get("user_guid")
+            r = ts.api.v2.users_delete(user_identifier=user_identifier)
+
+            if r.status_code == httpx.codes.NO_CONTENT:
+                log.info(f"Deleted user '{user_identifier}'")
+            else:
+                log.warning(f"Could not delete user '{user_identifier}' (HTTP {r.status_code}), see logs for details..")
+                log.debug(r.text)
+
+
 @app.command(dependencies=[thoughtspot])
 def sync(
     ctx: typer.Context,

cs_tools/cli/ux.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from collections.abc import Generator
 from typing import TYPE_CHECKING, Any, Callable, Optional
 import contextlib
 import logging
@@ -18,6 +19,19 @@
 rich_console = Console()
 
 
+@contextlib.contextmanager
+def _pause_live_for_debugging() -> Generator[None, None, None]:
+    """Pause live updates for debugging."""
+    if rich_console._live is not None:
+        rich_console._live.stop()
+
+    yield
+
+    if rich_console._live is not None:
+        rich_console.clear()
+        rich_console._live.start(refresh=True)
+
+
 class CSToolsCommand(typer.core.TyperCommand):
     """CSTools Commands can have dependencies."""