Crawler transform

.make.defaults

@@ -475,7 +475,7 @@ endif
 .defaults.test-src:: venv
 	@# Help: Run pytest on the test directory inside the venv 
 	source venv/bin/activate;       \
-	export PYTHONPATH=../src;       \
+	export PYTHONPATH=../src:../: ;  \
 	cd test; $(PYTEST) .
 
 # This is small convenience and the image itself must already be created.

transforms/.make.modules

@@ -0,0 +1,65 @@
+# Define the root of the local git clone for the common rules to be able 
+# know where they are running from.
+
+# Set this, before including .make.defaults, to 
+#   1 if requirements reference the latest code in the data processing library 
+#     in this repo (that is not yet published to pypi).	 This is the default setting.
+#   0 if the transforms DPK dependencies are on wheels published to 
+#     pypi (e.g. data-prep-toolkit=0.2.1)
+#USE_REPO_LIB_SRC=1
+
+# Include a library of common .transform.* targets which most
+# transforms should be able to reuse.  However, feel free
+# to override/redefine the rules below. 
+include $(REPOROOT)/transforms/.make.transforms
+
+######################################################################
+## Default setting for TRANSFORM_RUNTIME uses folder name-- Old layout
+TRANSFORM_RUNTIME=ray
+TRANSFORM_RUNTIME_SRC_FILE=-m dpk_$(TRANSFORM_NAME).$(TRANSFORM_RUNTIME).transform
+
+venv::	.transforms.ray-venv
+	source venv/bin/activate && $(PYTHON) -m pip install $(REPOROOT)/data-connector-lib
+
+test:: .transforms.test-src test-image
+
+clean:: .transforms.clean
+
+#image:: .transforms.ray-image
+
+test-src:: .transforms.test-src
+
+setup:: .transforms.setup
+
+publish:: publish-image
+
+publish-image:: .transforms.publish-image-ray
+
+test-image:: image .transforms.test-image-help .defaults.test-image-pytest .transforms.clean
+
+set-versions::
+
+## We need to think how we want to do this going forward 
+build::
+
+build-lib-wheel:
+	make -C $(REPOROOT)/data-processing-lib build-pkg-dist
+
+image:: build-lib-wheel
+	@$(eval LIB_WHEEL_FILE := $(shell find $(REPOROOT)/data-processing-lib/dist/*.whl))
+	rm -fr dist && mv $(REPOROOT)/data-processing-lib/dist .
+	$(eval WHEEL_FILE_NAME := $(shell basename $(LIB_WHEEL_FILE)))
+	$(DOCKER) build -t $(DOCKER_IMAGE_NAME) $(DOCKER_BUILD_EXTRA_ARGS)	\
+		--platform $(DOCKER_PLATFORM)  			\
+		--build-arg EXTRA_INDEX_URL=$(EXTRA_INDEX_URL)	\
+		--build-arg BASE_IMAGE=$(RAY_BASE_IMAGE)  		\
+		--build-arg BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')  	\
+		--build-arg WHEEL_FILE_NAME=$(WHEEL_FILE_NAME) \
+		--build-arg TRANSFORM_NAME=$(TRANSFORM_NAME) \
+		--build-arg GIT_COMMIT=$(shell git log -1 --format=%h) .
+	$(DOCKER) tag $(DOCKER_LOCAL_IMAGE)  $(DOCKER_REMOTE_IMAGE)
+	rm -fr dist
+
+publish:: publish-image
+
+

transforms/universal/web2parquet/Makefile

@@ -0,0 +1,23 @@
+REPOROOT=../../..
+# Use make help, to see the available rules
+include $(REPOROOT)/transforms/.make.modules
+
+#
+# This is intended to be included across the Makefiles provided within
+# a given transform's directory tree,  so must use compatible syntax.
+#
+################################################################################
+# This defines the name of the transform and is used to match against
+# expected files and is used to define the transform's image name. 
+TRANSFORM_NAME=$(shell basename `pwd`)
+
+################################################################################
+# This defines the transforms' version number as would be used
+# when publishing the wheel.  In general, only the micro version
+# number should be advanced relative to the DPK_VERSION. 
+#
+# If you change the versions numbers, be sure to run "make set-versions" to 
+# update version numbers across the transform (e.g., pyproject.toml).
+#TRANSFORM_VERSION=$(DPK_VERSION)
+
+

transforms/universal/web2parquet/dpk_web2parquet/config.py

@@ -0,0 +1,81 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+from argparse import ArgumentParser, Namespace
+
+from data_processing.transform import TransformConfiguration
+from data_processing.utils import CLIArgumentProvider
+from data_processing.utils import get_logger
+from dpk_web2parquet.transform import Web2ParquetTransform
+
+short_name = "web2parquet"
+cli_prefix = f"{short_name}_"
+urls_cli_param = f"{cli_prefix}urls"
+depth_cli_param = f"{cli_prefix}depth"
+downloads_cli_param = f"{cli_prefix}downloads"
+folder_cli_param = f"{cli_prefix}folder"
+
+
+logger = get_logger(__name__,"DEBUG")
+
+class Web2ParquetTransformConfiguration(TransformConfiguration):
+
+    """
+    Provides support for configuring and using the associated Transform class include
+    configuration with CLI args.
+    """
+
+    def __init__(self):
+        super().__init__(
+            name=short_name,
+            transform_class=Web2ParquetTransform
+        )
+
+    def add_input_params(self, parser: ArgumentParser) -> None:
+        """
+        Add Transform-specific arguments to the given  parser.
+        This will be included in a dictionary used to initialize the Web2ParquetTransform.
+        By convention a common prefix should be used for all transform-specific CLI args
+        (e.g, noop_, pii_, etc.)
+        """
+        parser.add_argument(f"--{depth_cli_param}", type=int, default=1,
+            help="maxumum depth relative to seed URL",
+        )
+        parser.add_argument(f"--{downloads_cli_param}", type=int, default=1,
+            help="maxumum number of downloaded URLs",
+        )
+        parser.add_argument(f"--{folder_cli_param}", type=str, default=None,
+            help="Folder where to store downloaded files",
+        )
+        parser.add_argument(f"--{urls_cli_param}", type=str, default=None,
+            help="List of Seed URLs for the crawler",
+        )
+
+    def apply_input_params(self, args: Namespace) -> bool:
+        """
+        Validate and apply the arguments that have been parsed
+        :param args: user defined arguments.
+        :return: True, if validate pass or False otherwise
+        """
+        captured = CLIArgumentProvider.capture_parameters(args, cli_prefix, False)
+        if captured.get("urls") is None:
+            logger.error(f"Parameter web2parquet_urls must specify a seed URL")
+            return False
+
+        self.params = self.params | captured
+        logger.info(f"web2parquet parameters are : {self.params}")
+        return True
+
+
+
+
+

transforms/universal/web2parquet/dpk_web2parquet/local.py

@@ -0,0 +1,26 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+
+from dpk_web2parquet.transform import Web2Parquet
+
+# create parameters
+
+if __name__ == "__main__":
+    # Here we show how to run outside of the runtime
+    # Create and configure the transform.
+    transform = Web2Parquet(urls= ['https://thealliance.ai/'],
+                                    depth=1, 
+                                    downloads=1)
+    table_list, metadata = transform.transform()
+    #print(f"\noutput table: {table_list}")
+    print(f"output metadata : {metadata}")

transforms/universal/web2parquet/dpk_web2parquet/local_python.py

@@ -0,0 +1,49 @@
+#(C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import os
+import sys
+
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.utils import ParamsUtils
+from dpk_web2parquet.python_runtime import Web2ParquetPythonTransformConfiguration
+
+
+# create parameters
+input_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..","test-data","input"))
+output_folder = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "test-data", "output"))
+local_conf = {
+    "input_folder": input_folder,
+    "output_folder": output_folder,
+}
+code_location = {"github": "github", "commit_hash": "12345", "path": "path"}
+params = {
+    # Data access. Only required parameters are specified
+    "data_local_config": ParamsUtils.convert_to_ast(local_conf),
+    # execution info
+    "runtime_pipeline_id": "pipeline_id",
+    "runtime_job_id": "job_id",
+    "runtime_code_location": ParamsUtils.convert_to_ast(code_location),
+    # web2parquet params
+    "web2parquet_urls": 'https://thealliance.ai/',
+    "web2parquet_depth": 1,
+    "web2parquet_downloads": 1,
+}
+
+
+if __name__ == "__main__":
+    # Set the simulated command line args
+    sys.argv = ParamsUtils.dict_to_req(d=params)
+    # create launcher
+    launcher = PythonTransformLauncher(runtime_config=Web2ParquetPythonTransformConfiguration())
+    # Launch the ray actor(s) to process the input
+    launcher.launch()

transforms/universal/web2parquet/dpk_web2parquet/python_runtime.py

@@ -0,0 +1,44 @@
+# (C) Copyright IBM Corp. 2024.
+# Licensed under the Apache License, Version 2.0 (the “License”);
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an “AS IS” BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+################################################################################
+
+import time
+
+from data_processing.runtime.pure_python import PythonTransformLauncher
+from data_processing.runtime.pure_python.runtime_configuration import (
+    PythonTransformRuntimeConfiguration,
+)
+from data_processing.utils import get_logger
+from dpk_web2parquet.config import Web2ParquetTransformConfiguration
+
+
+logger = get_logger(__name__)
+
+
+class Web2ParquetPythonTransformConfiguration(PythonTransformRuntimeConfiguration):
+    """
+    Implements the PythonTransformConfiguration for NOOP as required by the PythonTransformLauncher.
+    NOOP does not use a RayRuntime class so the superclass only needs the base
+    python-only configuration.
+    """
+
+    def __init__(self):
+        """
+        Initialization
+        :param base_configuration - base configuration class
+        """
+        super().__init__(transform_config=Web2ParquetTransformConfiguration())
+
+
+if __name__ == "__main__":
+    launcher = PythonTransformLauncher(Web2ParquetPythonTransformConfiguration())
+    logger.info("Launching web2parquet transform")
+    launcher.launch()