CE-633 Couchbase Plugin: Dsource sizing not working due to filesystem permissions

Author: ayesharustgi-delphix

plugin_config.yml

@@ -1,7 +1,7 @@
 id: 18f4ff11-b758-4bf2-9a37-719a22f5a4b8
 name: sc:couchbase
-externalVersion: "1.3.0"
-buildNumber: 1.3.0
+externalVersion: "1.3.1"
+buildNumber: 1.3.1
 language: PYTHON38
 hostTypes:
 - UNIX

src/controller/couchbase_operation.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020-2023 by Delphix. All rights reserved.
+# Copyright (c) 2020-2024 by Delphix. All rights reserved.
 #
 
 ##############################################################################
@@ -195,6 +195,9 @@ def run_os_command(self, os_command, **kwargs):
         stdout, stderr, exit_code = utilities.execute_bash(
             self.connection, command
         )
+        logger.debug(f"os_command stdout: {stdout}")
+        logger.debug(f"os_command stderr: {stderr}")
+        logger.debug(f"os_command exit_code: {exit_code}")
         return [stdout, stderr, exit_code]
 
     def restart_couchbase(self, provision=False):
@@ -430,6 +433,53 @@ def make_directory(self, directory_path, force_env_user=False):
 
         logger.debug("Changed the permission of directory")
 
+    def check_stale_mountpoint(self, path):
+
+        output, stderr, exit_code = self.run_os_command(
+            os_command="df", path=path
+        )
+        if exit_code != 0:
+            if "No such file or directory" in stderr:
+                # this is actually OK
+                return False
+            else:
+                logger.error(
+                    "df retured error - stale mount point or other error"
+                )
+                logger.error(
+                    "stdout: {} stderr: {} exit_code: {}".format(
+                        output, stderr, exit_code
+                    )
+                )
+                return True
+        else:
+            return False
+
+    def get_db_size(self, path: str) -> str:
+        """
+        Get the size of the dataset.
+
+        :param connection: Staging connection.
+        :param path: Mount location corresponding to dataset
+
+        :return: du command output.
+
+        """
+        logger.debug("Started db sizing")
+        du_std, du_stderr, du_exit_code = self.run_os_command(
+            os_command="du", mount_path=path
+        )
+        if du_exit_code != 0:
+            logger.error("Unable to calculate the dataset size")
+            logger.error(f"stderr: {du_stderr}")
+            raise UserError(
+                "Problem with measuring mounted file system",
+                "Ask OS admin to check mount",
+                du_stderr,
+            )
+        logger.debug(f"Completed db sizing {du_std}")
+        return du_std
+
     def create_config_dir(self):
         """create and return the hidden folder directory with name 'delphix'"""
 
@@ -494,8 +544,9 @@ def source_bucket_list(self):
 
     def get_backup_date(self, x):
         w = x.replace(
-            "{}/{}".format(
+            "{}/{}/{}".format(
                 self.parameters.couchbase_bak_loc,
+                self.parameters.archive_name,
                 self.parameters.couchbase_bak_repo,
             ),
             "",
@@ -1018,11 +1069,11 @@ def delete_config_folder(self):
                     srcname=config_directory_path,
                     trgname=target_folder,
                 )
-                logger.debug(
-                    f"mv directory >> command_output=={command_output}"
-                    f" , command_stderr=={command_stderr} , "
-                    f"command_exit_code=={command_exit_code}"
-                )
+                # logger.debug(
+                #     f"mv directory >> command_output=={command_output}"
+                #     f" , command_stderr=={command_stderr} , "
+                #     f"command_exit_code=={command_exit_code}"
+                # )
 
     def delete_xdcr_config(self):
         if self.parameters.d_source_type == "XDCR":

src/controller/helper_lib.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020-2023 by Delphix. All rights reserved.
+# Copyright (c) 2020-2024 by Delphix. All rights reserved.
 #
 
 ##############################################################################
@@ -34,7 +34,6 @@
 from internal_exceptions.plugin_exceptions import SourceConfigDiscoveryError
 from internal_exceptions.plugin_exceptions import UnmountFileSystemError
 from utils import utilities
-from dlpx.virtualization.common import RemoteConnection
 
 # Global logger object for this file
 logger = logging.getLogger(__name__)
@@ -524,27 +523,27 @@ def clean_stale_mountpoint(connection, path):
         )
 
 
-def get_db_size(connection: RemoteConnection, path: str) -> str:
-    """
-    Get the size of the dataset.
-
-    :param connection: Staging connection.
-    :param path: Mount location corresponding to dataset
-
-    :return: du command output.
-
-    """
-    logger.debug("Started db sizing")
-    du_std, du_stderr, du_exit_code = utilities.execute_bash(
-        connection, CommandFactory.du(mount_path=path)
-    )
-    if du_exit_code != 0:
-        logger.error("Unable to calculate the dataset size")
-        logger.error(f"stderr: {du_stderr}")
-        raise UserError(
-            "Problem with measuring mounted file system",
-            "Ask OS admin to check mount",
-            du_stderr,
-        )
-    logger.debug(f"Completed db sizing {du_std}")
-    return du_std
+# def get_db_size(connection: RemoteConnection, path: str) -> str:
+#     """
+#     Get the size of the dataset.
+#
+#     :param connection: Staging connection.
+#     :param path: Mount location corresponding to dataset
+#
+#     :return: du command output.
+#
+#     """
+#     logger.debug("Started db sizing")
+#     du_std, du_stderr, du_exit_code = utilities.execute_bash(
+#         connection, CommandFactory.du(mount_path=path)
+#     )
+#     if du_exit_code != 0:
+#         logger.error("Unable to calculate the dataset size")
+#         logger.error(f"stderr: {du_stderr}")
+#         raise UserError(
+#             "Problem with measuring mounted file system",
+#             "Ask OS admin to check mount",
+#             du_stderr,
+#         )
+#     logger.debug(f"Completed db sizing {du_std}")
+#     return du_std

src/db_commands/commands.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020-2023 by Delphix. All rights reserved.
+# Copyright (c) 2020-2024 by Delphix. All rights reserved.
 #
 
 ##############################################################################
@@ -90,8 +90,13 @@ def check_file(file_path, sudo=False, uid=None, **kwargs):
             )
 
     @staticmethod
-    def write_file(filename, data, **kwargs):
-        return "echo {data} > {filename}".format(filename=filename, data=data)
+    def write_file(filename, data, sudo=False, uid=None, **kwargs):
+        if sudo:
+            return f"sudo -u \#{uid} echo {data} > {filename}"
+        else:
+            return "echo {data} > {filename}".format(
+                filename=filename, data=data
+            )
 
     @staticmethod
     def get_ip_of_hostname(**kwargs):
@@ -196,11 +201,14 @@ def cat(path, sudo=False, uid=None, **kwargs):
             return "cat {path}".format(path=path)
 
     @staticmethod
-    def df(mount_path, **kwargs):
-        return "df -h {mount_path}".format(mount_path=mount_path)
+    def df(path, sudo=False, uid=None, **kwargs):
+        if sudo:
+            return f"sudo -u \#{uid} df -h {path}"
+        else:
+            return f"df -h {path}"
 
     @staticmethod
-    def mount(**kwargs):
+    def mount(sudo=False, uid=None, **kwargs):
         return "mount"
 
     @staticmethod
@@ -211,15 +219,21 @@ def resolve_name(hostname, **kwargs):
         )
 
     @staticmethod
-    def du(mount_path: str, **kwargs) -> str:
+    def du(mount_path: str, sudo=False, uid=None, **kwargs) -> str:
         """
         Returns command string to get size of dataset.
 
         :param mount_path: The path whose size is to be calculated
 
         :return: The du command string
         """
-        return f"du -s --block-size=1 --apparent-size {mount_path}"
+        if sudo:
+            return (
+                f"sudo -u \#{uid} du -s --block-size=1 --apparent-size "
+                f"{mount_path}"
+            )
+        else:
+            return f"du -s --block-size=1 --apparent-size {mount_path}"
 
 
 class DatabaseCommand(object):

src/operations/linked.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020-2023 by Delphix. All rights reserved.
+# Copyright (c) 2020-2024 by Delphix. All rights reserved.
 #
 #############################################################################
 # In this module, all dSource related operations are implemented
@@ -229,7 +229,7 @@ def _cleanup_in_exception_case(rx_connection, is_sync, is_snap_sync):
         raise
 
 
-def source_size(source_obj: StagedSource):
+def source_size(source_obj: StagedSource, repository, source_config):
     """
     Returns space occupied by the dataset on the mount point in bytes.
 
@@ -244,13 +244,18 @@ def source_size(source_obj: StagedSource):
         "Begin operation: Calculation of source"
         f" sizing for dSource {cluster_name}."
     )
+    srcsize_obj = CouchbaseOperation(
+        Resource.ObjectBuilder.set_staged_source(source_obj)
+        .set_repository(repository)
+        .build()
+    )
     try:
         if not helper_lib.check_stale_mountpoint(
             connection=connection, path=mount_path
         ) and helper_lib.check_server_is_used(
             connection=connection, path=mount_path
         ):
-            db_size_output = helper_lib.get_db_size(connection, mount_path)
+            db_size_output = srcsize_obj.get_db_size(path=mount_path)
             if db_size_output:
                 db_size = int(db_size_output.split()[0])
                 logger.debug(
@@ -267,8 +272,8 @@ def source_size(source_obj: StagedSource):
     except Exception as error:
         logger.debug("Exception: {}".format(str(error)))
         raise
-    finally:
-        logger.info(
-            "End operation: Calculation of source"
-            f" sizing for dSource {cluster_name} couldn't be completed."
-        )
+    # finally:
+    #     logger.info(
+    #         "End operation: Calculation of source"
+    #         f" sizing for dSource {cluster_name} couldn't be completed."
+    #     )

src/plugin_runner.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2020-2023 by Delphix. All rights reserved.
+# Copyright (c) 2020-2024 by Delphix. All rights reserved.
 #
 #
 
@@ -298,4 +298,4 @@ def add_node_to_virtual1(old_virtual_source):
 
 @plugin.linked.source_size()
 def linked_source_size(staged_source, repository, source_config):
-    return linked.source_size(staged_source)
+    return linked.source_size(staged_source, repository, source_config)