[dataset] columns and metrics API (nested)

requirements.txt

@@ -21,7 +21,7 @@ croniter==0.3.31          # via apache-superset (setup.py)
 cryptography==2.8         # via apache-superset (setup.py)
 decorator==4.4.1          # via retry
 defusedxml==0.6.0         # via python3-openid
-flask-appbuilder==2.3.0
+flask-appbuilder==2.3.0   # via apache-superset (setup.py)
 flask-babel==1.0.0        # via flask-appbuilder
 flask-caching==1.8.0
 flask-compress==1.4.0

superset/commands/exceptions.py

@@ -25,7 +25,10 @@
 class CommandException(SupersetException):
     """ Common base class for Command exceptions. """
 
-    pass
+    def __repr__(self):
+        if self._exception:
+            return self._exception
+        return self
 
 
 class CommandInvalidError(CommandException):

superset/connectors/sqla/models.py

@@ -1073,7 +1073,7 @@ def mutator(df: pd.DataFrame) -> None:
     def get_sqla_table_object(self) -> Table:
         return self.database.get_table(self.table_name, schema=self.schema)
 
-    def fetch_metadata(self) -> None:
+    def fetch_metadata(self, commit=True) -> None:
         """Fetches the metadata for the table and merges it in"""
         try:
             table = self.get_sqla_table_object()
@@ -1086,7 +1086,6 @@ def fetch_metadata(self) -> None:
                 ).format(self.table_name)
             )
 
-        M = SqlMetric
         metrics = []
         any_date_col = None
         db_engine_spec = self.database.db_engine_spec
@@ -1123,7 +1122,7 @@ def fetch_metadata(self) -> None:
                 any_date_col = col.name
 
         metrics.append(
-            M(
+            SqlMetric(
                 metric_name="count",
                 verbose_name="COUNT(*)",
                 metric_type="count",
@@ -1134,7 +1133,8 @@ def fetch_metadata(self) -> None:
             self.main_dttm_col = any_date_col
         self.add_missing_metrics(metrics)
         db.session.merge(self)
-        db.session.commit()
+        if commit:
+            db.session.commit()
 
     @classmethod
     def import_obj(cls, i_datasource, import_time=None) -> int:

superset/datasets/api.py

@@ -30,8 +30,10 @@
     DatasetForbiddenError,
     DatasetInvalidError,
     DatasetNotFoundError,
+    DatasetRefreshFailedError,
     DatasetUpdateFailedError,
 )
+from superset.datasets.commands.refresh import RefreshDatasetCommand
 from superset.datasets.commands.update import UpdateDatasetCommand
 from superset.datasets.schemas import DatasetPostSchema, DatasetPutSchema
 from superset.views.base import DatasourceFilter
@@ -49,9 +51,12 @@ class DatasetRestApi(BaseSupersetModelRestApi):
     allow_browser_login = True
 
     class_permission_name = "TableModelView"
-    include_route_methods = RouteMethod.REST_MODEL_VIEW_CRUD_SET | {RouteMethod.RELATED}
+    include_route_methods = (
+        RouteMethod.REST_MODEL_VIEW_CRUD_SET | {RouteMethod.RELATED} | {"refresh"}
+    )
 
     list_columns = [
+        "database_name",
         "changed_by_name",
         "changed_by_url",
         "changed_by.username",
@@ -79,6 +84,8 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         "template_params",
         "owners.id",
         "owners.username",
+        "columns",
+        "metrics",
     ]
     add_model_schema = DatasetPostSchema()
     edit_model_schema = DatasetPutSchema()
@@ -97,6 +104,8 @@ class DatasetRestApi(BaseSupersetModelRestApi):
         "is_sqllab_view",
         "template_params",
         "owners",
+        "columns",
+        "metrics",
     ]
     openapi_spec_tag = "Datasets"
 
@@ -268,3 +277,49 @@ def delete(self, pk: int) -> Response:  # pylint: disable=arguments-differ
         except DatasetDeleteFailedError as e:
             logger.error(f"Error deleting model {self.__class__.__name__}: {e}")
             return self.response_422(message=str(e))
+
+    @expose("/<pk>/refresh", methods=["PUT"])
+    @protect()
+    @safe
+    def refresh(self, pk: int) -> Response:  # pylint: disable=invalid-name
+        """Refresh a Dataset
+        ---
+        put:
+          description: >-
+            Refreshes and updates columns of a dataset
+          parameters:
+          - in: path
+            schema:
+              type: integer
+            name: pk
+          responses:
+            200:
+              description: Dataset delete
+              content:
+                application/json:
+                  schema:
+                    type: object
+                    properties:
+                      message:
+                        type: string
+            401:
+              $ref: '#/components/responses/401'
+            403:
+              $ref: '#/components/responses/403'
+            404:
+              $ref: '#/components/responses/404'
+            422:
+              $ref: '#/components/responses/422'
+            500:
+              $ref: '#/components/responses/500'
+        """
+        try:
+            RefreshDatasetCommand(g.user, pk).run()
+            return self.response(200, message="OK")
+        except DatasetNotFoundError:
+            return self.response_404()
+        except DatasetForbiddenError:
+            return self.response_403()
+        except DatasetRefreshFailedError as e:
+            logger.error(f"Error refreshing dataset {self.__class__.__name__}: {e}")
+            return self.response_422(message=str(e))

superset/datasets/commands/create.py

@@ -19,6 +19,7 @@
 
 from flask_appbuilder.security.sqla.models import User
 from marshmallow import ValidationError
+from sqlalchemy.exc import SQLAlchemyError
 
 from superset.commands.base import BaseCommand
 from superset.commands.utils import populate_owners
@@ -31,6 +32,7 @@
     TableNotFoundValidationError,
 )
 from superset.datasets.dao import DatasetDAO
+from superset.extensions import db, security_manager
 
 logger = logging.getLogger(__name__)
 
@@ -43,9 +45,23 @@ def __init__(self, user: User, data: Dict):
     def run(self):
         self.validate()
         try:
-            dataset = DatasetDAO.create(self._properties)
-        except DAOCreateFailedError as e:
-            logger.exception(e.exception)
+            # Creates SqlaTable (Dataset)
+            dataset = DatasetDAO.create(self._properties, commit=False)
+            # Updates columns and metrics from the dataset
+            dataset.fetch_metadata(commit=False)
+            # Add datasource access permission
+            security_manager.add_permission_view_menu(
+                "datasource_access", dataset.get_perm()
+            )
+            # Add schema access permission if exists
+            if dataset.schema:
+                security_manager.add_permission_view_menu(
+                    "schema_access", dataset.schema_perm
+                )
+            db.session.commit()
+        except (SQLAlchemyError, DAOCreateFailedError) as e:
+            logger.exception(e)
+            db.session.rollback()
             raise DatasetCreateFailedError()
         return dataset
 

superset/datasets/commands/delete.py

@@ -18,6 +18,7 @@
 from typing import Optional
 
 from flask_appbuilder.security.sqla.models import User
+from sqlalchemy.exc import SQLAlchemyError
 
 from superset.commands.base import BaseCommand
 from superset.connectors.sqla.models import SqlaTable
@@ -29,6 +30,7 @@
 )
 from superset.datasets.dao import DatasetDAO
 from superset.exceptions import SupersetSecurityException
+from superset.extensions import db, security_manager
 from superset.views.base import check_ownership
 
 logger = logging.getLogger(__name__)
@@ -43,9 +45,14 @@ def __init__(self, user: User, model_id: int):
     def run(self):
         self.validate()
         try:
-            dataset = DatasetDAO.delete(self._model)
-        except DAODeleteFailedError as e:
-            logger.exception(e.exception)
+            dataset = DatasetDAO.delete(self._model, commit=False)
+            security_manager.del_permission_view_menu(
+                "datasource_access", dataset.get_perm()
+            )
+            db.session.commit()
+        except (SQLAlchemyError, DAODeleteFailedError) as e:
+            logger.exception(e)
+            db.session.rollback()
             raise DatasetDeleteFailedError()
         return dataset
 

superset/datasets/commands/exceptions.py

@@ -57,6 +57,68 @@ def __init__(self, table_name: str):
         )
 
 
+class DatasetColumnNotFoundValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset column for update does not exist
+    """
+
+    def __init__(self):
+        super().__init__(_("One or more columns do not exist"), field_names=["columns"])
+
+
+class DatasetColumnsDuplicateValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset columns have a duplicate on the list
+    """
+
+    def __init__(self):
+        super().__init__(
+            _("One or more columns are duplicated"), field_names=["columns"]
+        )
+
+
+class DatasetColumnsExistsValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset columns already exist
+    """
+
+    def __init__(self):
+        super().__init__(
+            _("One or more columns already exist"), field_names=["columns"]
+        )
+
+
+class DatasetMetricsNotFoundValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset metric for update does not exist
+    """
+
+    def __init__(self):
+        super().__init__(_("One or more metrics do not exist"), field_names=["metrics"])
+
+
+class DatasetMetricsDuplicateValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset metrics have a duplicate on the list
+    """
+
+    def __init__(self):
+        super().__init__(
+            _("One or more metrics are duplicated"), field_names=["metrics"]
+        )
+
+
+class DatasetMetricsExistsValidationError(ValidationError):
+    """
+    Marshmallow validation error when dataset metrics already exist
+    """
+
+    def __init__(self):
+        super().__init__(
+            _("One or more metrics already exist"), field_names=["metrics"]
+        )
+
+
 class TableNotFoundValidationError(ValidationError):
     """
     Marshmallow validation error when a table does not exist on the database
@@ -99,5 +161,9 @@ class DatasetDeleteFailedError(DeleteFailedError):
     message = _("Dataset could not be deleted.")
 
 
+class DatasetRefreshFailedError(UpdateFailedError):
+    message = _("Dataset could not be updated.")
+
+
 class DatasetForbiddenError(ForbiddenError):
     message = _("Changing this dataset is forbidden")

superset/datasets/commands/refresh.py

@@ -0,0 +1,61 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import logging
+from typing import Optional
+
+from flask_appbuilder.security.sqla.models import User
+
+from superset.commands.base import BaseCommand
+from superset.connectors.sqla.models import SqlaTable
+from superset.datasets.commands.exceptions import (
+    DatasetForbiddenError,
+    DatasetNotFoundError,
+    DatasetRefreshFailedError,
+)
+from superset.datasets.dao import DatasetDAO
+from superset.exceptions import SupersetSecurityException
+from superset.views.base import check_ownership
+
+logger = logging.getLogger(__name__)
+
+
+class RefreshDatasetCommand(BaseCommand):
+    def __init__(self, user: User, model_id: int):
+        self._actor = user
+        self._model_id = model_id
+        self._model: Optional[SqlaTable] = None
+
+    def run(self):
+        self.validate()
+        try:
+            # Updates columns and metrics from the dataset
+            self._model.fetch_metadata()
+        except Exception as e:
+            logger.exception(e)
+            raise DatasetRefreshFailedError()
+        return self._model
+
+    def validate(self) -> None:
+        # Validate/populate model exists
+        self._model = DatasetDAO.find_by_id(self._model_id)
+        if not self._model:
+            raise DatasetNotFoundError()
+        # Check ownership
+        try:
+            check_ownership(self._model)
+        except SupersetSecurityException:
+            raise DatasetForbiddenError()