Spark.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNs6njWUNhCFicBSRemPR2P",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/NXHM/spark_platzi_exam/blob/main/Spark.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "[Añadir repositorios](https://medium.com/analytics-vidhya/ultimate-guide-for-setting-up-pyspark-in-google-colab-7637f697daf1)"
      ],
      "metadata": {
        "id": "dByrJYqrRswP"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Version 2.7"
      ],
      "metadata": {
        "id": "WpU05a6rXWWQ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Start here! Select spark version\n",
        "ver_spark = '3.1.2'\n",
        "\n",
        "# Install Java JDK\n",
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "\n",
        "# Download Spark version\n",
        "import os\n",
        "\n",
        "os.system(f\"wget -q https://archive.apache.org/dist/spark/spark-{ver_spark}/spark-{ver_spark}-bin-hadoop2.7.tgz\")\n",
        "os.system(f\"tar xf spark-{ver_spark}-bin-hadoop2.7.tgz\")\n",
        "\n",
        "# Define environment variables\n",
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = f\"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = f\"/content/spark-{ver_spark}-bin-hadoop2.7\"\n",
        "\n",
        "# Install findspark: it will locate Spark on the system and import it as a regular library.\n",
        "!pip install -q findspark\n",
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "# Create a SparkSession\n",
        "from pyspark.sql import SparkSession\n",
        "spark = SparkSession.builder\\\n",
        "        .master(\"local\")\\\n",
        "        .appName(\"Colab\")\\\n",
        "        .config('spark.ui.port', '4050')\\\n",
        "        .getOrCreate()"
      ],
      "metadata": {
        "id": "mZ5IbUnPXLYF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Version 3"
      ],
      "metadata": {
        "id": "lqFZU4PZXZAO"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bopXyv7lQJ4p",
        "outputId": "3c9a6c84-7401-49d5-facf-d5d55609a43a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "spark-3.5.1-bin-hadoop3/\n",
            "spark-3.5.1-bin-hadoop3/sbin/\n",
            "spark-3.5.1-bin-hadoop3/sbin/spark-config.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-slave.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-mesos-dispatcher.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-workers.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-slaves.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-all.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-all.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/workers.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-mesos-dispatcher.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/spark-daemon.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/decommission-worker.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/slaves.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-mesos-shuffle-service.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-history-server.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-worker.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/decommission-slave.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-thriftserver.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-worker.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-slaves.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-connect-server.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-thriftserver.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-history-server.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-connect-server.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/spark-daemons.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-slave.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-master.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/start-mesos-shuffle-service.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-master.sh\n",
            "spark-3.5.1-bin-hadoop3/sbin/stop-workers.sh\n",
            "spark-3.5.1-bin-hadoop3/licenses/\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-automaton.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-d3.min.js.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-scopt.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-leveldbjni.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-blas.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-spire.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-paranamer.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-matchMedia-polyfill.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jaxb-runtime.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-zstd-jni.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-pyrolite.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-dagre-d3.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-javax-transaction-transaction-api.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-json-formatter.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jsp-api.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-arpack.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-AnchorJS.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-JLargeArrays.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-protobuf.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-pmml-model.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-slf4j.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-py4j.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-datatables.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-join.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-bootstrap.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jakarta.activation-api.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jline.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-vis-timeline.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jakarta-annotation-api\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-zstd.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-respond.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jakarta-ws-rs-api\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-graphlib-dot.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-modernizr.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-kryo.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-dnsjava.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-cloudpickle.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-janino.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-sbt-launch-lib.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-javolution.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-mustache.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-f2j.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jodd.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-machinist.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-JTransforms.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-CC0.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jakarta.xml.bind-api.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-reflectasm.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-sorttable.js.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-istack-commons-runtime.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-minlog.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-jquery.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-antlr.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-xmlenc.txt\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-javassist.html\n",
            "spark-3.5.1-bin-hadoop3/licenses/LICENSE-re2j.txt\n",
            "spark-3.5.1-bin-hadoop3/data/\n",
            "spark-3.5.1-bin-hadoop3/data/graphx/\n",
            "spark-3.5.1-bin-hadoop3/data/graphx/followers.txt\n",
            "spark-3.5.1-bin-hadoop3/data/graphx/users.txt\n",
            "spark-3.5.1-bin-hadoop3/data/streaming/\n",
            "spark-3.5.1-bin-hadoop3/data/streaming/AFINN-111.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_multiclass_classification_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_svm_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/kmeans_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/streaming_kmeans_data_test.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_movielens_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/29.5.a_b_EGDP022204.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/54893.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/DP802813.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/not-image.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/kittens/DP153539.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/multi-channel/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/multi-channel/BGRA_alpha_60.png\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/multi-channel/chr30.4.184.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/multi-channel/BGRA.png\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/multi-channel/grayscale.jpg\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/origin/license.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/images/license.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/ridge-data/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/ridge-data/lpsa.data\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/als/\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/als/sample_movielens_ratings.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/als/test.data\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_binary_classification_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/gmm_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_libsvm_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/pagerank_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_isotonic_regression_libsvm_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_linear_regression_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_lda_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_fpgrowth.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_kmeans_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/sample_lda_libsvm_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/mllib/pic_data.txt\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/smallJar.jar\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/junitLargeJar.jar\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/crc/\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/crc/smallJar.txt\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/crc/junitLargeJar.txt\n",
            "spark-3.5.1-bin-hadoop3/data/artifact-tests/crc/README.md\n",
            "spark-3.5.1-bin-hadoop3/jars/\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-container-servlet-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-sketch_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-annotations-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-x86_64.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spire-macros_2.12-0.17.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-extensions-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jline-2.14.6.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/orc-mapreduce-1.9.2-shaded-protobuf.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/HikariCP-2.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-storageclass-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/orc-core-1.9.2-shaded-protobuf.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/objenesis-3.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/snakeyaml-2.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-hadoop-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/httpclient-4.5.14.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/curator-client-2.13.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/json4s-core_2.12-3.7.0-M11.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-classes-epoll-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-aarch_64.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-io-2.13.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-databind-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-network-common_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hadoop-yarn-server-web-proxy-3.3.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-codec-socks-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hadoop-client-api-3.3.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/curator-recipes-2.13.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/RoaringBitmap-0.9.45.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-mllib-local_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-hive-thriftserver_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-math3-3.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/janino-3.1.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/datanucleus-rdbms-4.1.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-catalyst_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-exec-2.3.9-core.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-mesos_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/gson-2.2.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-node-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-cli-1.5.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-launcher_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/shims-0.9.45.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spire_2.12-0.17.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-handler-proxy-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/algebra_2.12-2.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/log4j-slf4j2-impl-2.20.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/json4s-scalap_2.12-3.7.0-M11.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/ST4-4.0.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/aircompressor-0.26.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-column-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-kubernetes_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-scheduling-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/snappy-java-1.1.10.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/transaction-api-1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-common-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-core-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-certificates-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.ws.rs-api-2.1.6.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/lapack-3.0.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-codec-http2-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-native-epoll-4.1.96.Final-linux-aarch_64.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/JTransforms-3.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-llap-common-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/json-1.8.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-storage-api-2.8.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jsr305-3.0.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-common-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/oro-2.0.8.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/cats-kernel_2.12-2.1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/breeze-macros_2.12-2.1.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/py4j-0.10.9.7.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-collections4-4.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/javolution-5.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/slf4j-api-2.0.7.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-network-shuffle_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/tink-1.9.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arrow-memory-netty-12.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/avro-1.11.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-networking-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-metrics-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/leveldbjni-all-1.8.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-core-asl-1.9.13.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-shims-common-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/audience-annotations-0.5.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-parser-combinators_2.12-2.3.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/pickle-1.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-service-rpc-3.1.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-batch-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spire-platform_2.12-0.17.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-common-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/blas-3.0.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-codec-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/log4j-api-2.20.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-dbcp-1.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/chill_2.12-0.10.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-compiler-3.1.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-common-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/metrics-json-4.2.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-codec-1.16.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kryo-shaded-4.0.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-native-kqueue-4.1.96.Final-osx-x86_64.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/univocity-parsers-2.9.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/json4s-jackson_2.12-3.7.0-M11.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-client-api-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/okio-1.15.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-resource-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/mesos-1.4.3-shaded-protobuf.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-apiextensions-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-metastore-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-reflect-2.12.18.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/flatbuffers-java-1.12.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/ivy-2.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-hk2-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/breeze_2.12-2.1.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-common-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-jackson-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hk2-api-2.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arpack-3.0.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-xml_2.12-2.1.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-core_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/lz4-java-1.8.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/joda-time-2.12.5.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/metrics-jvm-4.2.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/log4j-1.2-api-2.20.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-coordination-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-client-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/json4s-ast_2.12-3.7.0-M11.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-apps-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-lang-2.6.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-gatewayapi-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-mapper-asl-1.9.13.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-common-utils_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/zstd-jni-1.5.5-4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jodd-core-3.5.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jpam-1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/metrics-core-4.2.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-unsafe_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.servlet-api-4.0.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-format-structures-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/log4j-core-2.20.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/rocksdbjni-8.3.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-shims-scheduler-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-codec-http-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/paranamer-2.8.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-shims-0.23-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-autoscaling-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-collection-compat_2.12-2.7.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/dropwizard-metrics-hadoop-metrics2-reporter-0.1.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/curator-framework-2.13.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-mllib_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/opencsv-2.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-buffer-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-library-2.12.18.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/xz-1.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-collections-3.2.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-dataformat-yaml-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/datanucleus-core-4.1.17.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-repl_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/bonecp-0.8.0.RELEASE.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/guava-14.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/zjsonpatch-0.3.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-sql-api_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-all-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-streaming_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arrow-format-12.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/libthrift-0.12.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-hive_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-resolver-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hadoop-shaded-guava-1.1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-httpclient-okhttp-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-shims-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/metrics-jmx-4.2.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-beeline-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.annotation-api-1.3.5.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-rbac-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/metrics-graphite-4.2.19.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/activation-1.1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jcl-over-slf4j-2.0.7.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/okhttp-3.12.12.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/httpcore-4.4.16.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-flowcontrol-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-tags_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-text-1.10.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-client-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/javassist-3.29.2-GA.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/parquet-encoding-1.13.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-pool-1.5.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/zookeeper-jute-3.6.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/xbean-asm9-shaded-4.23.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-native-unix-common-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/minlog-1.3.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/avro-mapred-1.11.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/stax-api-1.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.xml.bind-api-2.3.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-module-scala_2.12-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arrow-vector-12.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/antlr-runtime-3.5.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/snakeyaml-engine-2.6.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hk2-locator-2.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-logging-1.1.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-jdbc-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-cli-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arrow-memory-core-12.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-discovery-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/logging-interceptor-3.12.12.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/JLargeArrays-1.5.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/derby-10.14.2.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hadoop-client-runtime-3.3.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-kvstore_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-compress-1.23.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jul-to-slf4j-2.0.7.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-graphx_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-admissionregistration-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-lang3-3.12.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hive-serde-2.3.9.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/javax.jdo-3.2.0-m3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/arpack_combined_all-0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/super-csv-2.2.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/hk2-utils-2.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jackson-datatype-jsr310-2.15.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/antlr4-runtime-4.9.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/stream-2.9.6.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/threeten-extra-1.7.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spire-util_2.12-0.17.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.validation-api-2.0.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jaxb-runtime-2.3.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/scala-compiler-2.12.18.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/datasketches-memory-2.1.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-yarn_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-events-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-core-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/aopalliance-repackaged-2.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/avro-ipc-1.11.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jta-1.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/compress-lzf-1.1.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/annotations-17.0.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/datanucleus-api-jdo-4.2.4.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-server-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/commons-crypto-1.1.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jdo-api-3.0.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jersey-container-servlet-core-2.40.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-handler-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/jakarta.inject-2.6.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/chill-java-0.10.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/netty-transport-classes-kqueue-4.1.96.Final.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/istack-commons-runtime-3.0.8.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/spark-sql_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/zookeeper-3.6.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/orc-shims-1.9.2.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/datasketches-java-3.3.0.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/libfb303-0.9.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/osgi-resource-locator-1.0.3.jar\n",
            "spark-3.5.1-bin-hadoop3/jars/kubernetes-model-policy-6.7.2.jar\n",
            "spark-3.5.1-bin-hadoop3/examples/\n",
            "spark-3.5.1-bin-hadoop3/examples/jars/\n",
            "spark-3.5.1-bin-hadoop3/examples/jars/spark-examples_2.12-3.5.1.jar\n",
            "spark-3.5.1-bin-hadoop3/examples/jars/scopt_2.12-3.7.1.jar\n",
            "spark-3.5.1-bin-hadoop3/examples/src/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/LiveJournalPageRank.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/SynthBenchmark.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/Analytics.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/SSSPExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/AggregateMessagesExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/ComprehensiveExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/PageRankExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/ConnectedComponentsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/graphx/TriangleCountingExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkALS.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/SqlNetworkWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/CustomReceiver.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/StreamingExamples.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKafkaWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/DirectKerberizedKafkaWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/HdfsWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/RecoverableNetworkWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/RawNetworkGrep.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/StatefulNetworkWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/NetworkWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/QueueStream.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewGenerator.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/streaming/clickstream/PageViewStream.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ExceptionHandlingTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkPageRank.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/MiniReadWriteTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/KMeansExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/SummarizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BucketedRandomProjectionLSHExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RFormulaExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NGramExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NormalizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LDAExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MaxAbsScalerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionWithElasticNetExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StringIndexerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FeatureHasherExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/EstimatorTransformerParamExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/NaiveBayesExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ImputerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PipelineExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionWithElasticNetExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MinHashLSHExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FPGrowthExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorAssemblerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeClassifierExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/InteractionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeClassificationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/OneVsRestExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PolynomialExpansionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/OneHotEncoderExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VarianceThresholdSelectorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ChiSquareTestExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/IsotonicRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DeveloperApiExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BisectingKMeansExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearSVCExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LogisticRegressionSummaryExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MinMaxScalerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MulticlassLogisticRegressionWithElasticNetExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StopWordsRemoverExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/IndexToStringExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/TokenizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/CountVectorizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/QuantileDiscretizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/TfIdfExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RobustScalerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GaussianMixtureExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorIndexerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GBTExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaCrossValidationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GeneralizedLinearRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PrefixSpanExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/SQLTransformerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorSlicerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/GradientBoostedTreeRegressorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/CorrelationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/UnivariateFeatureSelectorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DecisionTreeRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ElementwiseProductExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestRegressorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/AFTSurvivalRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/RandomForestClassifierExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/StandardScalerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ChiSqSelectorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BinarizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/VectorSizeHintExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/LinearRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/DCTExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/Word2VecExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/MultilayerPerceptronClassifierExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FMClassifierExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/FMRegressorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PowerIterationClusteringExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/PCAExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/BucketizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/ml/ModelSelectionViaTrainValidationSplitExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SQLDataSourceExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredSessionization.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredKafkaWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredKerberizedKafkaWordCount.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredNetworkWordCountWindowed.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/streaming/StructuredComplexSessionization.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SimpleTypedAggregator.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedTypedAggregation.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedUntypedAggregation.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/SparkSQLExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/hive/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/hive/SparkHiveExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/RDDRelation.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/jdbc/ExampleJdbcConnectionProvider.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/sql/UserDefinedScalar.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalFileLR.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalPi.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalLR.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SkewedGroupByTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkPi.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkHdfsLR.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SimpleSkewedGroupByTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/KMeansExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnySVD.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLogisticRegression.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/NormalizerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DenseKMeans.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LDAExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StratifiedSamplingExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRunner.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnSourceVectorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/NaiveBayesExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingLinearRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LogisticRegressionWithLBFGSExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/AbstractParams.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/FPGrowthExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/AssociationRulesExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SummaryStatisticsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LBFGSExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MultiLabelMetricsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeClassificationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/CorrelationsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MultivariateSummarizer.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingKMeansExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MovieLensALS.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PMMLModelExportExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/IsotonicRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BisectingKMeansExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SimpleFPGrowth.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomRDDGeneration.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RecommendationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TFIDFExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SVMWithSGDExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RandomForestClassificationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostedTreesRunner.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PCAOnRowMatrixExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SampledRDDs.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingKolmogorovSmirnovTestExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GaussianMixtureExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StreamingTestExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/RankingMetricsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassification.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PrefixSpanExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/HypothesisTestingExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/KernelDensityEstimationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/DecisionTreeRegressionExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/ElementwiseProductExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/TallSkinnyPCA.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/Correlations.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/BinaryClassificationMetricsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/CosineSimilarity.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SVDExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/StandardScalerExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/ChiSqSelectorExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/SparseNaiveBayes.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/MulticlassMetricsExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/Word2VecExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/GradientBoostingClassificationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/LatentDirichletAllocationExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/mllib/PowerIterationClusteringExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/pythonconverters/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/pythonconverters/AvroConverters.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/BroadcastTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/HdfsTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalALS.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithoutLoader.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SparkSessionExtensionsTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/AgeExample.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/extensions/SessionExtensionsWithLoader.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkKMeans.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LocalKMeans.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkLR.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkRemoteFileTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/DFSReadWriteTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/SparkTC.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/GroupByTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/DriverSubmissionTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/LogQuery.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/MultiBroadcastTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scala/org/apache/spark/examples/AccumulatorMetricsTest.scala\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/streaming/structured_network_wordcount.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/lm_with_elastic_net.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/kstest.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/fmRegressor.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/survreg.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/randomForest.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/prefixSpan.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/kmeans.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/powerIterationClustering.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/fpm.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/gbt.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/fmClassifier.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/logit.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/als.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/glm.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/bisectingKmeans.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/decisionTree.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/ml.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/naiveBayes.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/isoreg.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/svmLinear.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/mlp.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/gaussianMixture.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/ml/lda.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/dataframe.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/RSparkSQLExample.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/r/data-manipulation.R\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scripts/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/scripts/getGpusResources.sh\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaHdfsLR.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKerberizedKafkaWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaStatefulNetworkWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaDirectKafkaWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaCustomReceiver.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecord.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaNetworkWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaQueueStream.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaRecoverableNetworkWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/streaming/JavaSqlNetworkWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaStatusTrackerDemo.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPolynomialExpansionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionSummaryExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearRegressionWithElasticNetExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeClassifierExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSizeHintExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFPGrowthExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestRegressorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStopWordsRemoverExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketedRandomProjectionLSHExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaTrainValidationSplitExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSqSelectorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaOneHotEncoderExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPrefixSpanExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStringIndexerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGaussianMixtureExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMulticlassLogisticRegressionWithElasticNetExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorSlicerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLogisticRegressionWithElasticNetExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBinarizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPipelineExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRandomForestClassifierExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaChiSquareTestExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaStandardScalerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaModelSelectionViaCrossValidationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaElementwiseProductExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFeatureHasherExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMinMaxScalerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaInteractionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRFormulaExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMaxAbsScalerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaSummarizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaIndexToStringExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaRobustScalerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaIsotonicRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaImputerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaTokenizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLDAExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMultilayerPerceptronClassifierExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaEstimatorTransformerParamExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaCorrelationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaKMeansExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDocument.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaMinHashLSHExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFMClassifierExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPowerIterationClusteringExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNGramExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNormalizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorAssemblerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGradientBoostedTreeRegressorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBucketizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaQuantileDiscretizerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLabeledDocument.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaUnivariateFeatureSelectorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaGeneralizedLinearRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaPCAExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaAFTSurvivalRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVarianceThresholdSelectorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaOneVsRestExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaLinearSVCExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaNaiveBayesExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaFMRegressorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaVectorIndexerExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaTfIdfExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaDCTExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/ml/JavaBisectingKMeansExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaLogQuery.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCountWindowed.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredKerberizedKafkaWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredSessionization.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredNetworkWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredKafkaWordCount.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/streaming/JavaStructuredComplexSessionization.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaSQLDataSourceExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedScalar.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/hive/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/hive/JavaSparkHiveExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedUntypedAggregation.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaUserDefinedTypedAggregation.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/sql/JavaSparkSQLExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaPageRank.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVMWithSGDExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaALS.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaStreamingTestExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaChiSqSelectorExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaMulticlassClassificationMetricsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaCorrelationsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaElementwiseProductExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaBinaryClassificationMetricsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaStratifiedSamplingExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSummaryStatisticsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLogisticRegressionWithLBFGSExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaLBFGSExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPowerIterationClusteringExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeClassificationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaPCAExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaKernelDensityEstimationExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaDecisionTreeRegressionExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaHypothesisTestingKolmogorovSmirnovTestExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/mllib/JavaBisectingKMeansExample.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaTC.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/java/org/apache/spark/examples/JavaSparkPi.java\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/dir1/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/dir1/file1.parquet\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/dir1/file3.json\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/dir1/dir2/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/dir1/dir2/file2.parquet\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/users.avro\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/users.parquet\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/META-INF/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/META-INF/services/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/META-INF/services/org.apache.spark.sql.SparkSessionExtensionsProvider\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/META-INF/services/org.apache.spark.sql.jdbc.JdbcConnectionProvider\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/people.txt\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/employees.json\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/kv1.txt\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/user.avsc\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/users.orc\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/people.json\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/full_user.avsc\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/resources/people.csv\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/hdfs_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/sql_network_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/network_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/recoverable_network_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/stateful_network_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/network_wordjoinsentiments.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/streaming/queue_stream.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/avro_inputformat.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/transitive_closure.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/stopwords_remover_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/pipeline_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/vector_assembler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/__init__,py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/cross_validator.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/logistic_regression_with_elastic_net.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/tf_idf_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/summarizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/string_indexer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/decision_tree_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/sql_transformer.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/interaction_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/count_vectorizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/dct_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/power_iteration_clustering_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/quantile_discretizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/bucketizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/rformula_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/gradient_boosted_tree_regressor_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/isotonic_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/tokenizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/one_vs_rest_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/lda_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/chisq_selector_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/feature_hasher_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/estimator_transformer_param_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/univariate_feature_selector_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/imputer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/vector_slicer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/chi_square_test_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/decision_tree_classification_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/min_max_scaler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/gradient_boosted_tree_classifier_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/prefixspan_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/standard_scaler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/aft_survival_regression.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/bisecting_k_means_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/vector_indexer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/robust_scaler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/vector_size_hint_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/n_gram_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/als_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/binarizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/variance_threshold_selector_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/fm_regressor_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/kmeans_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/normalizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/dataframe_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/multiclass_logistic_regression_with_elastic_net.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/max_abs_scaler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/elementwise_product_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/linearsvc.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/random_forest_classifier_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/correlation_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/multilayer_perceptron_classification.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/naive_bayes_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/fm_classifier_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/logistic_regression_summary_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/index_to_string_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/onehot_encoder_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/polynomial_expansion_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/train_validation_split.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/gaussian_mixture_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/min_hash_lsh_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/pca_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/fpgrowth_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/linear_regression_with_elastic_net.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/random_forest_regressor_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/bucketed_random_projection_lsh_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/word2vec_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/ml/generalized_linear_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/datasource.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/__init__,py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount_session_window.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount_windowed.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/structured_network_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/structured_sessionization.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/streaming/structured_kafka_wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/arrow.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/udtf.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/basic.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/hive.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sql/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/wordcount.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/pagerank.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/kmeans.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/tf_idf_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/kernel_density_estimation_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/decision_tree_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/random_forest_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/random_forest_classification_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/sampled_rdds.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/recommendation_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/power_iteration_clustering_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/linear_regression_with_sgd_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/latent_dirichlet_allocation_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/isotonic_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/correlations_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/logistic_regression_with_lbfgs_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/gradient_boosting_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/ranking_metrics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/binary_classification_metrics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/hypothesis_testing_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/regression_metrics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/decision_tree_classification_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/random_rdd_generation.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/hypothesis_testing_kolmogorov_smirnov_test_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/multi_label_metrics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/streaming_linear_regression_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/summary_statistics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/standard_scaler_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/bisecting_k_means_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/stratified_sampling_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/kmeans.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/word2vec.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/gaussian_mixture_model.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/normalizer_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/svd_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/pca_rowmatrix_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/elementwise_product_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/k_means_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/naive_bayes_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/multi_class_metrics_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/gradient_boosting_classification_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/svm_with_sgd_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/streaming_k_means_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/gaussian_mixture_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/correlations.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/logistic_regression.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/fpgrowth_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/mllib/word2vec_example.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/sort.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/pi.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/als.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/status_api_demo.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/logistic_regression.py\n",
            "spark-3.5.1-bin-hadoop3/examples/src/main/python/parquet_inputformat.py\n",
            "spark-3.5.1-bin-hadoop3/NOTICE\n",
            "spark-3.5.1-bin-hadoop3/conf/\n",
            "spark-3.5.1-bin-hadoop3/conf/log4j2.properties.template\n",
            "spark-3.5.1-bin-hadoop3/conf/workers.template\n",
            "spark-3.5.1-bin-hadoop3/conf/spark-env.sh.template\n",
            "spark-3.5.1-bin-hadoop3/conf/metrics.properties.template\n",
            "spark-3.5.1-bin-hadoop3/conf/spark-defaults.conf.template\n",
            "spark-3.5.1-bin-hadoop3/conf/fairscheduler.xml.template\n",
            "spark-3.5.1-bin-hadoop3/bin/\n",
            "spark-3.5.1-bin-hadoop3/bin/run-example\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-shell.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-class\n",
            "spark-3.5.1-bin-hadoop3/bin/sparkR\n",
            "spark-3.5.1-bin-hadoop3/bin/beeline\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-shell2.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/find-spark-home\n",
            "spark-3.5.1-bin-hadoop3/bin/pyspark.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-submit2.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/pyspark2.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-connect-shell\n",
            "spark-3.5.1-bin-hadoop3/bin/docker-image-tool.sh\n",
            "spark-3.5.1-bin-hadoop3/bin/load-spark-env.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-sql.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/find-spark-home.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/beeline.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-class.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/sparkR2.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-sql2.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-sql\n",
            "spark-3.5.1-bin-hadoop3/bin/sparkR.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-submit\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-shell\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-submit.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/pyspark\n",
            "spark-3.5.1-bin-hadoop3/bin/run-example.cmd\n",
            "spark-3.5.1-bin-hadoop3/bin/load-spark-env.sh\n",
            "spark-3.5.1-bin-hadoop3/bin/spark-class2.cmd\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/decommissioning.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/autoscale.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/python_executable_check.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/worker_memory_check.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/py_container_checks.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/pyfiles.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/tests/decommissioning_cleanup.py\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/decom.sh\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/Dockerfile\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/entrypoint.sh\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/python/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/python/Dockerfile\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/R/\n",
            "spark-3.5.1-bin-hadoop3/kubernetes/dockerfiles/spark/bindings/R/Dockerfile\n",
            "spark-3.5.1-bin-hadoop3/README.md\n",
            "spark-3.5.1-bin-hadoop3/LICENSE\n",
            "spark-3.5.1-bin-hadoop3/yarn/\n",
            "spark-3.5.1-bin-hadoop3/yarn/spark-3.5.1-yarn-shuffle.jar\n",
            "spark-3.5.1-bin-hadoop3/python/\n",
            "spark-3.5.1-bin-hadoop3/python/setup.py\n",
            "spark-3.5.1-bin-hadoop3/python/MANIFEST.in\n",
            "spark-3.5.1-bin-hadoop3/python/test_coverage/\n",
            "spark-3.5.1-bin-hadoop3/python/test_coverage/sitecustomize.py\n",
            "spark-3.5.1-bin-hadoop3/python/test_coverage/conf/\n",
            "spark-3.5.1-bin-hadoop3/python/test_coverage/conf/spark-defaults.conf\n",
            "spark-3.5.1-bin-hadoop3/python/test_coverage/coverage_daemon.py\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/hello/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/hello/hello.txt\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/hello/sub_hello/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/hello/sub_hello/sub_hello.txt\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/userlib-0.1.zip\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/test_pytorch_training_file.py\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/people_array.json\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/streaming/text-test.txt\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/_SUCCESS\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=0/c=0/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/.part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc.crc\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/orc_partitioned/b=1/c=1/part-r-00000-829af031-b970-49d6-ad39-30460a0be2c8.orc\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/text-test.txt\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/people_array_utf16le.json\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/people1.json\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/ages_newlines.csv\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/sql/people.json\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/SimpleHTTPServer.py\n",
            "spark-3.5.1-bin-hadoop3/python/test_support/userlibrary.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/SOURCES.txt\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/top_level.txt\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/PKG-INFO\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/requires.txt\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark.egg-info/dependency_links.txt\n",
            "spark-3.5.1-bin-hadoop3/python/run-tests.py\n",
            "spark-3.5.1-bin-hadoop3/python/setup.cfg\n",
            "spark-3.5.1-bin-hadoop3/python/run-tests\n",
            "spark-3.5.1-bin-hadoop3/python/lib/\n",
            "spark-3.5.1-bin-hadoop3/python/lib/py4j-0.10.9.7-src.zip\n",
            "spark-3.5.1-bin-hadoop3/python/lib/PY4J_LICENSE.txt\n",
            "spark-3.5.1-bin-hadoop3/python/lib/pyspark.zip\n",
            "spark-3.5.1-bin-hadoop3/python/.coveragerc\n",
            "spark-3.5.1-bin-hadoop3/python/docs/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/make.bat\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/quickstart_connect.ipynb\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/testing_pyspark.ipynb\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/install.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/quickstart_ps.ipynb\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/quickstart_df.ipynb\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/getting_started/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/arrow_pandas.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/types.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/from_to_dbms.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/best_practices.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/typehints.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/faq.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/options.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/pandas_pyspark.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/pandas_on_spark/transform_apply.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/sql/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/sql/arrow_pandas.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/sql/python_udtf.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/sql/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/python_packaging.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/user_guide/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/migration_guide/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/migration_guide/pyspark_upgrade.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/migration_guide/koalas_to_pyspark.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/migration_guide/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_templates/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_templates/version-switcher.html\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_templates/autosummary/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_templates/autosummary/class.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_templates/autosummary/class_with_docs.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/debugging.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/setting_ide.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/errors.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/testing.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/development/contributing.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.mllib.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ss/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ss/query_management.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ss/core_classes.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ss/io.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ss/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.errors.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/grouping.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/udtf.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/udf.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/observation.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/core_classes.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/io.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/dataframe.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/row.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/catalog.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/avro.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/protobuf.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/configuration.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/spark_session.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/window.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/column.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/functions.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.sql/data_types.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.streaming.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.resource.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.testing.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/ml.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/general_functions.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/indexing.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/extensions.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/io.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/series.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/frame.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/groupby.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/resampling.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/window.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.pandas/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/reference/pyspark.ml.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/index.rst\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_static/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_static/css/\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_static/css/pyspark.css\n",
            "spark-3.5.1-bin-hadoop3/python/docs/source/_static/versions.json\n",
            "spark-3.5.1-bin-hadoop3/python/docs/Makefile\n",
            "spark-3.5.1-bin-hadoop3/python/docs/make2.bat\n",
            "spark-3.5.1-bin-hadoop3/python/.gitignore\n",
            "spark-3.5.1-bin-hadoop3/python/dist/\n",
            "spark-3.5.1-bin-hadoop3/python/mypy.ini\n",
            "spark-3.5.1-bin-hadoop3/python/README.md\n",
            "spark-3.5.1-bin-hadoop3/python/run-tests-with-coverage\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resultiterable.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/status.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/statcounter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/rdd.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/kinesis.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/test_context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/test_listener.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/test_dstream.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/test_kinesis.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/listener.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/dstream.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/requests.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/profile.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/tests/test_resources.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/information.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/resource/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/join.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/dl_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/recommendation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/model_cache.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/tuning.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/summarizer.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/classification.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/connect/io_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/regression.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/wrapper.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/common.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/clustering.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/param/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/param/_shared_params_code_gen.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/param/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/param/shared.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/_typing.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_training_summary.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_classification.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_classification.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_tuning.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_tuning.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_summarizer.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_parity_torch_distributor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_legacy_mode_summarizer.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_function.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_connect_pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/connect/test_parity_torch_data_loader.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_persistence.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_dl_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_algorithms.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_image.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_evaluation.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_param.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_readable.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_clustering.yaml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_regression.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_feature.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/typing/test_classification.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_tvs_io_nested.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_cv_io_basic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_tuning.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_cv_io_nested.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_tvs_io_basic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_tvs_io_pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/tuning/test_cv_io_pipeline.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_linalg.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_param.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_wrapper.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tests/test_model_cache.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tuning.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/linalg/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/linalg/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/image.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/tests/test_data_loader.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/tests/test_log_communication.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/tests/test_distributor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/distributor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/torch_run_process_wrapper.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/log_communication.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/torch/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/classification.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/deepspeed/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/deepspeed/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/deepspeed/tests/test_deepspeed_distributor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/deepspeed/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/deepspeed/deepspeed_distributor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/fpm.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/ml/tree.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/sql_formatter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/expressions_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/commands_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/example_plugins_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/types_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/example_plugins_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/commands_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/common_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/relations_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/relations_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/base_pb2_grpc.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/expressions_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/common_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/catalog_pb2.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/types_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/proto/catalog_pb2.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/column.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/query.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/worker/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/worker/listener_worker.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/worker/foreach_batch_worker.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/worker/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/plan.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/client/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/client/artifact.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/client/reattach.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/client/core.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/client/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/session.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/group.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/udtf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/_typing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/avro/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/avro/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/avro/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/protobuf/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/protobuf/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/protobuf/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/catalog.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/expressions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/dataframe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/types.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/connect/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/column.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/query.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/state.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/listener.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/observation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/_typing.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/test_parity_streaming.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/test_parity_foreach_batch.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/test_parity_foreach.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/streaming/test_parity_listener.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map_with_state.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_datasources.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_arrow_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_catalog.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_session.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_udf_grouped_agg.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/client/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/client/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/client/test_client.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/client/test_artifact.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_serde.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_basic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_plan.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_types.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_arrow.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_arrow_python_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_udf_scalar.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_grouped_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_column.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_cogrouped_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_errors.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_column.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_udf_window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_connect_function.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_udtf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_dataframe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_group.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/connect/test_parity_pandas_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_serde.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_arrow_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_pandas_sqlmetrics.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_udtf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming_listener.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming_foreach_batch.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/streaming/test_streaming_foreach.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_group.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_datasources.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_catalog.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_errors.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_session.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_arrow.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_dataframe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_column.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_dataframe.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_session.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_functions.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_udf.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/typing/test_readwriter.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_cogrouped_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_grouped_map_with_state.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_grouped_agg.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_grouped_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_scalar.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_typehints_with_future_annotations.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_udf_window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/pandas/test_pandas_map.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_types.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_arrow_python_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_column.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_udf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_udf_profiler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/tests/test_conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/session.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/group.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/udtf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/group_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/map_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/typehints.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/series.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/frame.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/protocols/__init__.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/_typing/__init__.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/functions.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/serializers.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/types.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/pandas/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/avro/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/avro/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/avro/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/protobuf/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/protobuf/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/protobuf/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/catalog.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/readwriter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/dataframe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/types.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/sql/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/accumulators.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/rddsampler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/_typing.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/py.typed\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_rddbarrier.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_context.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_profiler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_broadcast.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_pin_thread.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_rdd.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_readwrite.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/typing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/typing/test_resultiterable.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/typing/test_context.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/typing/test_rdd.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/typing/test_core.yml\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_rddsampler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_appsubmit.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_daemon.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_shuffle.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_taskcontext.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_serializers.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_worker.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_memory_profiler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_install_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_conf.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_statcounter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_stage_sched.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/tests/test_join.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/storagelevel.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/profiler.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/version.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/instrumentation_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/worker_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/worker.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/cloudpickle/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/cloudpickle/compat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/cloudpickle/cloudpickle_fast.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/cloudpickle/cloudpickle.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/cloudpickle/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/pandasutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/mlutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/sqlutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/streamingutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/connectutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/testing/mllibutils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/__pycache__/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/__pycache__/install.cpython-38.pyc\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/sql_formatter.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/categorical.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/namespace.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/accessors.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/usage_logging/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/usage_logging/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/usage_logging/usage_logger.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/correlation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/resample.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/plot/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/plot/plotly.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/plot/core.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/plot/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/plot/matplotlib.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/supported_api_gen.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/null_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/num_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/udt_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/string_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/complex_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/date_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/categorical_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/timedelta_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/datetime_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/binary_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/boolean_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/data_type_ops/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_rolling.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_sql.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_dataframe_spark_io.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_apply_func.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_compute.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_any_all.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_melt.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_pivot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_cov.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_describe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_corrwith.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_combine.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_eval.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/computation/test_parity_binary_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_apply_func.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_split_apply.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_describe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_aggregate.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_head_tail.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/groupby/test_parity_groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_basic_slow.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_cov_corrwith.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_align.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_dot_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/test_parity_setitem_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/diff_frames_ops/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_attrs.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_constructor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_reindexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_truncate.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_take.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_reshaping.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/frame/test_parity_time_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_extension.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/io/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/io/test_parity_io.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/io/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_plotly.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_plotly.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot_matplotlib.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot_matplotlib.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_frame_plot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/test_parity_series_plot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/plot/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_indexops_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_typedef.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_spark_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_stats.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_dataframe_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_default_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_date_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_null_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_string_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_categorical_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_timedelta_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_complex_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_datetime_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_arithmetic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_boolean_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_udt_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/testing_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_binary_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/data_type_ops/test_parity_num_reverse.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_expanding.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_category.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_timedelta.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_reindex.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_reset_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_align.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_indexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_datetime.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/indexes/test_parity_rename.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_series_string.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_generic_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_ops_on_diff_frames_groupby_rolling.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_series_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_numpy_compat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_compute.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_sort.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_all_any.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_as_type.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_arg_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_as_of.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/series/test_parity_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_series_datetime.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_ewm.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_config.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_indexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_internal.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_reshape.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_resample.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_frame_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_categorical.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_rolling.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_scalars.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_expanding.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_repr.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_csv.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/connect/test_parity_namespace.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_combine.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_pivot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_any_all.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_melt.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_apply_func.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_eval.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_corrwith.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_cov.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_describe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_compute.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/computation/test_binary_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_series_string.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_typedef.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_ewm.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_apply_func.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_head_tail.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_split_apply.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_describe.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/groupby/test_aggregate.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_dot_frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_dot_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_align.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_cov_corrwith.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_setitem_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/diff_frames_ops/test_basic_slow.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_time_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_take.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_constructor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_reshaping.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_attrs.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_reindexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_truncate.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/frame/test_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_numpy_compat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_scalars.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_sql.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_frame_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/io/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/io/test_io.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/io/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_extension.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot_matplotlib.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot_plotly.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_series_plot_matplotlib.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot_plotly.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/plot/test_frame_plot.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_datetime_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_timedelta_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_num_reverse.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_complex_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_boolean_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_num_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/testing_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_null_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_num_arithmetic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_string_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_categorical_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_binary_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_udt_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/data_type_ops/test_date_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_spark_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_resample.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_expanding.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_namespace.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_align.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_reset_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_rename.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_reindex.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_category.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_indexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_timedelta.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/indexes/test_datetime.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_expanding.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_arg_ops.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_as_type.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_cumulative.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_all_any.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_as_of.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_missing_data.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_compute.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/series/test_sort.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_default_index.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_repr.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_dataframe_spark_io.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_internal.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_series_datetime.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_generic_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_reshape.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_csv.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_stats.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_config.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_ops_on_diff_frames_groupby_rolling.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_series_conversion.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_indexing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_indexops_spark.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/tests/test_categorical.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/internal.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/timedelta.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/datetimes.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/category.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/numeric.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/indexes/multi.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/numpy_compat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/datetimes.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/frame.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/groupby.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/common.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/resample.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/indexes.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/general_functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/missing/scalars.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/_typing.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/spark/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/spark/accessors.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/spark/functions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/spark/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/spark/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/series.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/window.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/exceptions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/mlflow.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/extensions.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/generic.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/config.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/strings.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/typedef/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/typedef/typehints.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/typedef/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/pandas/sql_processor.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/recommendation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/regression.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/common.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/clustering.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/_typing.pyi\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_streaming_algorithms.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_algorithms.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_util.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_linalg.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/test_stat.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/linalg/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/linalg/distributed.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/linalg/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/evaluation.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/distribution.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/KernelDensity.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/test.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/stat/_statistics.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/classification.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/feature.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/fpm.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/tree.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/mllib/random.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/broadcast.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/daemon.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/traceback_utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/serializers.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/tests/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/tests/test_errors.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/tests/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/error_classes.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/utils.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/exceptions/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/exceptions/connect.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/exceptions/captured.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/exceptions/base.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/exceptions/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/errors/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/shell.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/python/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/python/pyspark/\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/python/pyspark/shell.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/taskcontext.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/files.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/__init__.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/install.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/_globals.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/shuffle.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/java_gateway.py\n",
            "spark-3.5.1-bin-hadoop3/python/pyspark/find_spark_home.py\n",
            "spark-3.5.1-bin-hadoop3/RELEASE\n",
            "spark-3.5.1-bin-hadoop3/R/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/nsInfo.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/hsearch.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/vignette.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/Rd.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/package.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/links.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/Meta/features.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/doc/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/doc/index.html\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.html\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.Rmd\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/doc/sparkr-vignettes.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/SparkR.rdx\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/paths.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/SparkR.rdb\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/AnIndex\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/help/aliases.rds\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/NAMESPACE\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/tests/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/tests/testthat/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/tests/testthat/test_basic.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/INDEX\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/DESCRIPTION\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/profile/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/profile/general.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/profile/shell.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/worker/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/worker/daemon.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/worker/worker.R\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/html/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/html/00Index.html\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/html/R.css\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/R/\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/R/SparkR.rdx\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/R/SparkR.rdb\n",
            "spark-3.5.1-bin-hadoop3/R/lib/SparkR/R/SparkR\n",
            "spark-3.5.1-bin-hadoop3/R/lib/sparkr.zip\n",
            "Requirement already satisfied: pyspark in /usr/local/lib/python3.10/dist-packages (3.5.1)\n",
            "Requirement already satisfied: py4j==0.10.9.7 in /usr/local/lib/python3.10/dist-packages (from pyspark) (0.10.9.7)\n"
          ]
        }
      ],
      "source": [
        "!apt-get install openjdk-8-jdk-headless -qq > /dev/null\n",
        "!wget -q https://dlcdn.apache.org/spark/spark-3.5.1/spark-3.5.1-bin-hadoop3.tgz\n",
        "!tar -xvf spark-3.5.1-bin-hadoop3.tgz\n",
        "!pip install -q findspark\n",
        "!pip install pyspark"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!file spark-3.5.1-bin-hadoop3.tgz"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CvFC_7vUWua1",
        "outputId": "2418bde4-6616-4341-adc3-d81d8071abdc"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "spark-3.5.1-bin-hadoop3.tgz: gzip compressed data, from Unix, original size modulo 2^32 445542400\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ[\"JAVA_HOME\"] = \"/usr/lib/jvm/java-8-openjdk-amd64\"\n",
        "os.environ[\"SPARK_HOME\"] = \"/content/spark-3.5.1-bin-hadoop3\""
      ],
      "metadata": {
        "id": "38y-XpG3SUDe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#from pydrive2.auth import GoogleAuth\n",
        "#from pydrive2.drive import GoogleDrive\n",
        "#from google.colab import auth\n",
        "#from oauth2client.client import GoogleCredentials\n",
        "\n",
        "from pyspark import SparkContext #mecanismos que tenemos para interactuar con SPark\n",
        "from pyspark.sql import SparkSession # engloba todos los contextos: mejores formas de configuracion\n"
      ],
      "metadata": {
        "id": "yPF3KmvZa4dm"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import findspark\n",
        "findspark.init()\n",
        "\n",
        "spark = SparkSession.builder\\\n",
        "  .master(\"local\")\\\n",
        "  .appName(\"primeraSesion\")\\\n",
        "  .getOrCreate()"
      ],
      "metadata": {
        "id": "LkmItfCOSXzo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "2F2ReW9bcna6",
        "outputId": "ce34213a-a97c-4ad5-887c-300c6e5ad5fb"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7e2766d35420>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://4480aeb724f1:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.5.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>primerContexto</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark.stop() #detiene la sesion"
      ],
      "metadata": {
        "id": "kd8019nzcSdO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Crear con contexto y sesion"
      ],
      "metadata": {
        "id": "XJpeXYnXcn86"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sc= SparkContext(master =\"local\",appName=\"primerContexto\")"
      ],
      "metadata": {
        "id": "l6i3-GV9cY4L"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark2 = SparkSession(sc)"
      ],
      "metadata": {
        "id": "u_vaU2L4cnDU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "spark2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "id": "r-50b1AKcxCC",
        "outputId": "91fc6257-bbdf-4b23-f9fc-2fd3b7a86625"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.session.SparkSession at 0x7e2766d35420>"
            ],
            "text/html": [
              "\n",
              "            <div>\n",
              "                <p><b>SparkSession - in-memory</b></p>\n",
              "                \n",
              "        <div>\n",
              "            <p><b>SparkContext</b></p>\n",
              "\n",
              "            <p><a href=\"http://4480aeb724f1:4040\">Spark UI</a></p>\n",
              "\n",
              "            <dl>\n",
              "              <dt>Version</dt>\n",
              "                <dd><code>v3.5.1</code></dd>\n",
              "              <dt>Master</dt>\n",
              "                <dd><code>local</code></dd>\n",
              "              <dt>AppName</dt>\n",
              "                <dd><code>primerContexto</code></dd>\n",
              "            </dl>\n",
              "        </div>\n",
              "        \n",
              "            </div>\n",
              "        "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark2.stop()"
      ],
      "metadata": {
        "id": "RiYMN3xafOxY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Transformacion y accion"
      ],
      "metadata": {
        "id": "W3Aq6rwMfHk6"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sc= SparkContext(master =\"local\",appName=\"transformacionesYAcciones\")"
      ],
      "metadata": {
        "id": "dorGWjKFfKDn"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd1 = sc.parallelize([1,2,3])"
      ],
      "metadata": {
        "id": "LZYTWsuZfWCT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "type(rdd1)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 174
        },
        "id": "wwv5mCD6fkky",
        "outputId": "ed8d81fd-c56a-4514-bc42-366c6679d8ec"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pyspark.rdd.RDD"
            ],
            "text/html": [
              "<div style=\"max-width:800px; border: 1px solid var(--colab-border-color);\"><style>\n",
              "      pre.function-repr-contents {\n",
              "        overflow-x: auto;\n",
              "        padding: 8px 12px;\n",
              "        max-height: 500px;\n",
              "      }\n",
              "\n",
              "      pre.function-repr-contents.function-repr-contents-collapsed {\n",
              "        cursor: pointer;\n",
              "        max-height: 100px;\n",
              "      }\n",
              "    </style>\n",
              "    <pre style=\"white-space: initial; background:\n",
              "         var(--colab-secondary-surface-color); padding: 8px 12px;\n",
              "         border-bottom: 1px solid var(--colab-border-color);\"><b>pyspark.rdd.RDD</b><br/>def __init__(jrdd: &#x27;JavaObject&#x27;, ctx: &#x27;SparkContext&#x27;, jrdd_deserializer: Serializer=AutoBatchedSerializer(CPickleSerializer()))</pre><pre class=\"function-repr-contents function-repr-contents-collapsed\" style=\"\"><a class=\"filepath\" style=\"display:none\" href=\"#\">/content/spark-3.5.1-bin-hadoop3/python/pyspark/rdd.py</a>A Resilient Distributed Dataset (RDD), the basic abstraction in Spark.\n",
              "Represents an immutable, partitioned collection of elements that can be\n",
              "operated on in parallel.</pre>\n",
              "      <script>\n",
              "      if (google.colab.kernel.accessAllowed && google.colab.files && google.colab.files.view) {\n",
              "        for (const element of document.querySelectorAll('.filepath')) {\n",
              "          element.style.display = 'block'\n",
              "          element.onclick = (event) => {\n",
              "            event.preventDefault();\n",
              "            event.stopPropagation();\n",
              "            google.colab.files.view(element.textContent, 336);\n",
              "          };\n",
              "        }\n",
              "      }\n",
              "      for (const element of document.querySelectorAll('.function-repr-contents')) {\n",
              "        element.onclick = (event) => {\n",
              "          event.preventDefault();\n",
              "          event.stopPropagation();\n",
              "          element.classList.toggle('function-repr-contents-collapsed');\n",
              "        };\n",
              "      }\n",
              "      </script>\n",
              "      </div>"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd1.collect()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pN-oSnG0ftbT",
        "outputId": "4b2c8c9e-92fd-426b-b6eb-5dc51c0f8217"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[1, 2, 3]"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Usar la data del curso desde drive"
      ],
      "metadata": {
        "id": "riWhN4X3au3i"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import auth\n",
        "from googleapiclient.discovery import build\n",
        "from googleapiclient.errors import HttpError\n",
        "from google.colab import drive\n",
        "import pandas as pd\n",
        "drive.mount('/content/drive') # autentica el usuario para Drive\n",
        "\n",
        "auth.authenticate_user() # autentica el usuario para la API\n",
        "\n",
        "# Pregunta el email\n",
        "try:\n",
        "    service = build('oauth2', 'v2', cache_discovery=False) # No me funcionaba con la libreria oauth2\n",
        "    user_info = service.userinfo().get().execute()\n",
        "    user_email = user_info['email']\n",
        "    #print(user_email)\n",
        "except HttpError as e:\n",
        "    print(f'An error occurred: {e}')\n",
        "# Segun el usuario ingresa a su drive y elige el path\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8F4YxlU8Yc4n",
        "outputId": "71b45ced-0e2f-4568-a870-38c1598006bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path = '/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/data/files/'\n"
      ],
      "metadata": {
        "id": "4SLYA5sAkti5"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD2 = sc.textFile(path+\"paises.csv\")\\\n",
        "  .map(lambda line : line.split(\",\"))"
      ],
      "metadata": {
        "id": "i6J2U76YjTCo"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD2.take(15)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QX3CjGK3jXcE",
        "outputId": "483bf33a-980f-455a-f45d-20b27060ee67"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['id', 'equipo', 'sigla'],\n",
              " ['1', '30. Februar', 'AUT'],\n",
              " ['2', 'A North American Team', 'MEX'],\n",
              " ['3', 'Acipactli', 'MEX'],\n",
              " ['4', 'Acturus', 'ARG'],\n",
              " ['5', 'Afghanistan', 'AFG'],\n",
              " ['6', 'Akatonbo', 'IRL'],\n",
              " ['7', 'Alain IV', 'SUI'],\n",
              " ['8', 'Albania', 'ALB'],\n",
              " ['9', 'Alcaid', 'POR'],\n",
              " ['10', 'Alcyon-6', 'FRA'],\n",
              " ['11', 'Alcyon-7', 'FRA'],\n",
              " ['12', 'Aldebaran', 'ITA'],\n",
              " ['13', 'Aldebaran II', 'ITA'],\n",
              " ['14', 'Aletta', 'IRL']]"
            ]
          },
          "metadata": {},
          "execution_count": 346
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD = sc.textFile(path+\"paises.csv\")\\\n",
        "  .map(lambda line : line.split(\",\"))"
      ],
      "metadata": {
        "id": "AqFf5N_ei5Il"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.take(15)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FVaTJ_1vkRAI",
        "outputId": "1627e442-6439-4970-fd5c-83f407e9dfa0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['id', 'equipo', 'sigla'],\n",
              " ['1', '30. Februar', 'AUT'],\n",
              " ['2', 'A North American Team', 'MEX'],\n",
              " ['3', 'Acipactli', 'MEX'],\n",
              " ['4', 'Acturus', 'ARG'],\n",
              " ['5', 'Afghanistan', 'AFG'],\n",
              " ['6', 'Akatonbo', 'IRL'],\n",
              " ['7', 'Alain IV', 'SUI'],\n",
              " ['8', 'Albania', 'ALB'],\n",
              " ['9', 'Alcaid', 'POR'],\n",
              " ['10', 'Alcyon-6', 'FRA'],\n",
              " ['11', 'Alcyon-7', 'FRA'],\n",
              " ['12', 'Aldebaran', 'ITA'],\n",
              " ['13', 'Aldebaran II', 'ITA'],\n",
              " ['14', 'Aletta', 'IRL']]"
            ]
          },
          "metadata": {},
          "execution_count": 162
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Usar tuplas, porque internamente de llave:valor se hace tuplas\n",
        "equiposOlimpicosRDD.map(lambda x: (x[2])).distinct().count()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xjIO2W7-ecpw",
        "outputId": "40076671-d746-4e9f-e232-f740dfe7e7c5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "231"
            ]
          },
          "metadata": {},
          "execution_count": 163
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# key, valor\n",
        "equiposOlimpicosRDD.map(lambda x: (x[2], x[1])).groupByKey()\\\n",
        "  .mapValues(len).take(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zoyqmm-Ylor4",
        "outputId": "86ea1b9a-b031-405f-df6e-684050081b9e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('sigla', 1), ('AUT', 11), ('MEX', 9), ('ARG', 18), ('AFG', 1)]"
            ]
          },
          "metadata": {},
          "execution_count": 164
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.map(lambda x: (x[2], x[1])).groupByKey()\\\n",
        "  .mapValues(list).take(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dgocSSv-mU4r",
        "outputId": "f1143aa4-d0ad-4867-839c-05f3ad7e4843"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('sigla', ['equipo']),\n",
              " ('AUT',\n",
              "  ['30. Februar',\n",
              "   'Austria',\n",
              "   'Austria-1',\n",
              "   'Austria-2',\n",
              "   'Breslau',\n",
              "   'Brigantia',\n",
              "   'Donar III',\n",
              "   'Evita VI',\n",
              "   'May-Be 1960',\n",
              "   '\"R.-V. Germania; Leitmeritz\"',\n",
              "   'Surprise']),\n",
              " ('MEX',\n",
              "  ['A North American Team',\n",
              "   'Acipactli',\n",
              "   'Chamukina',\n",
              "   'Mexico',\n",
              "   'Mexico-1',\n",
              "   'Mexico-2',\n",
              "   'Nausikaa 4',\n",
              "   'Tlaloc',\n",
              "   'Xolotl']),\n",
              " ('ARG',\n",
              "  ['Acturus',\n",
              "   'Antares',\n",
              "   'Arcturus',\n",
              "   'Ardilla',\n",
              "   'Argentina',\n",
              "   'Argentina-1',\n",
              "   'Argentina-2',\n",
              "   'Blue Red',\n",
              "   'Covunco III',\n",
              "   'Cupidon III',\n",
              "   'Djinn',\n",
              "   'Gullvinge',\n",
              "   'Matrero II',\n",
              "   'Mizar',\n",
              "   'Pampero',\n",
              "   'Rampage',\n",
              "   'Tango',\n",
              "   'Wiking']),\n",
              " ('AFG', ['Afghanistan'])]"
            ]
          },
          "metadata": {},
          "execution_count": 165
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposArgentinos = equiposOlimpicosRDD.filter(lambda l: \"ARG\" in l)\n",
        "equiposArgentinos.collect() # Todos los RDDs con los valores requeridos, son traidos"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_YKv2Ogympw8",
        "outputId": "f6b23d82-f1df-458c-95ba-43efed593f67"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['4', 'Acturus', 'ARG'],\n",
              " ['37', 'Antares', 'ARG'],\n",
              " ['42', 'Arcturus', 'ARG'],\n",
              " ['43', 'Ardilla', 'ARG'],\n",
              " ['45', 'Argentina', 'ARG'],\n",
              " ['46', 'Argentina-1', 'ARG'],\n",
              " ['47', 'Argentina-2', 'ARG'],\n",
              " ['119', 'Blue Red', 'ARG'],\n",
              " ['238', 'Covunco III', 'ARG'],\n",
              " ['252', 'Cupidon III', 'ARG'],\n",
              " ['288', 'Djinn', 'ARG'],\n",
              " ['436', 'Gullvinge', 'ARG'],\n",
              " ['644', 'Matrero II', 'ARG'],\n",
              " ['672', 'Mizar', 'ARG'],\n",
              " ['774', 'Pampero', 'ARG'],\n",
              " ['843', 'Rampage', 'ARG'],\n",
              " ['1031', 'Tango', 'ARG'],\n",
              " ['1162', 'Wiking', 'ARG']]"
            ]
          },
          "metadata": {},
          "execution_count": 166
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.count() # Cuenta todo, no es optimo"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bU9t6RehnUmR",
        "outputId": "88ccbded-70a1-45d3-d9b9-a55688a815b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1185"
            ]
          },
          "metadata": {},
          "execution_count": 167
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.countApprox(20)# cuenta hasta el tiempo entregado en milisegundos"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LFrcrAjHmpnJ",
        "outputId": "ec1ea80a-3145-4409-ddef-2f27268e5f87"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1185"
            ]
          },
          "metadata": {},
          "execution_count": 168
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD = sc.textFile(path+\"deportista.csv\")\\\n",
        "  .map( lambda l: l.split(\",\"))\n",
        "deportistaOlimpicoRDD2 = sc.textFile(path+\"deportista2.csv\")\\\n",
        "  .map( lambda l: l.split(\",\"))\n"
      ],
      "metadata": {
        "id": "9t3ENp5Rmpex"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD=deportistaOlimpicoRDD\\\n",
        "  .union(deportistaOlimpicoRDD2) #quita duplicados"
      ],
      "metadata": {
        "id": "q_bwr-7SmpXb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.count() # SE usa, porque son pocos datos. !Solo con pocos"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fJwycg3ioARJ",
        "outputId": "f360f509-c9af-48fa-f436-29936a440856"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "135572"
            ]
          },
          "metadata": {},
          "execution_count": 172
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.top(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L0bWAgpYoabR",
        "outputId": "c1f32246-e3f8-4604-9327-8b4bfa13eff7"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['id', 'equipo', 'sigla'], ['999', 'Stella-2', 'NOR']]"
            ]
          },
          "metadata": {},
          "execution_count": 173
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.top(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "miA6t5b7oAOF",
        "outputId": "3306b52b-cea0-4fbe-a361-93964e3d7598"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['deportista_id', 'nombre', 'genero', 'edad', 'altura', 'peso', 'equipo_id'],\n",
              " ['99999', 'Alexander Grant Alick Rennie', '1', '32', '182', '71', '967']]"
            ]
          },
          "metadata": {},
          "execution_count": 174
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.map(lambda l: [l[-1], l[:-1]])\\\n",
        "  .join(equiposOlimpicosRDD.map(lambda x: [x[0],x[2]]))\\\n",
        "  .take(6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hL4n-B_HmpFh",
        "outputId": "3180567a-53af-4cac-a27d-a76dbf1e7cf2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('199', (['1', 'A Dijiang', '1', '24', '180', '80'], 'CHN')),\n",
              " ('199', (['2', 'A Lamusi', '1', '23', '170', '60'], 'CHN')),\n",
              " ('199', (['602', 'Abudoureheman', '1', '22', '182', '75'], 'CHN')),\n",
              " ('199', (['1463', 'Ai Linuer', '1', '25', '160', '62'], 'CHN')),\n",
              " ('199', (['1464', 'Ai Yanhan', '2', '14', '168', '54'], 'CHN')),\n",
              " ('199', (['3605', 'An Weijiang', '1', '22', '178', '72'], 'CHN'))]"
            ]
          },
          "metadata": {},
          "execution_count": 175
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.map(lambda l: [l[-1], l[:-1]])\\\n",
        "  .join(equiposOlimpicosRDD.map(lambda x: [x[0],x[2]]))\\\n",
        "  .takeSample(False,6,25)#valores repetidos, cuantos valores, seed"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VYgsR3mfpSuA",
        "outputId": "50a558bc-a22b-4aa9-a834-5e7fe24fa344"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('362', (['131505', 'Steven Woodburn', '1', '24', '185', '90'], 'FRA')),\n",
              " ('967', (['13626', 'Jill Brresen', '2', '22', '170', '57'], 'RSA')),\n",
              " ('482', (['44299', 'Gumundur Gumundsson', '1', '23', '174', '77'], 'ISL')),\n",
              " ('970', (['68062', 'Lee MinHui', '2', '28', '174', '65'], 'KOR')),\n",
              " ('794', (['92442', 'Luis Paz Zoldan', '1', '19', '187', '82'], 'PER')),\n",
              " ('413', (['26822', 'Jared Mark Deacon', '1', '24', '185', '77'], 'GBR'))]"
            ]
          },
          "metadata": {},
          "execution_count": 176
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultado = sc.textFile(path+\"resultados.csv\")\\\n",
        "  .map(lambda l: l.split(\",\"))"
      ],
      "metadata": {
        "id": "6INdA-YWpc86"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resultadoGanador = resultado.filter(lambda l: \"NA\" not in l[1])"
      ],
      "metadata": {
        "id": "2UGe02fqpcyT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resultadoGanador.take(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "btXp_RkCpctc",
        "outputId": "689aafd1-d12c-4939-fe8c-5a5d6c5cf3c9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['resultado_id', 'medalla', 'deportista_id', 'juego_id', 'evento_id'],\n",
              " ['4', 'Gold', '4', '2', '4']]"
            ]
          },
          "metadata": {},
          "execution_count": 179
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Juntar resultado con  equipo"
      ],
      "metadata": {
        "id": "e3MxdzBordAt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "equiposOlimpicosRDD.top(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ESbI1hDQr8q7",
        "outputId": "65c2a60f-d927-4b36-befb-2fb243842603"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['id', 'equipo', 'sigla'], ['999', 'Stella-2', 'NOR']]"
            ]
          },
          "metadata": {},
          "execution_count": 180
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.top(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2Vq11st9r424",
        "outputId": "ea96bcfa-3f97-40a0-a197-5ca4e0b19556"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['deportista_id', 'nombre', 'genero', 'edad', 'altura', 'peso', 'equipo_id'],\n",
              " ['99999', 'Alexander Grant Alick Rennie', '1', '32', '182', '71', '967']]"
            ]
          },
          "metadata": {},
          "execution_count": 181
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultadoGanador.take(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0k3_AITPsL6E",
        "outputId": "26fe0e80-a01e-4195-a6f4-6e5978829e03"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['resultado_id', 'medalla', 'deportista_id', 'juego_id', 'evento_id'],\n",
              " ['4', 'Gold', '4', '2', '4']]"
            ]
          },
          "metadata": {},
          "execution_count": 182
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#El id de deportista en l[1][0][0],\n",
        "#el valor es el id del equipos l[0]\n",
        "# y el resto de la información del deportista (l[1][0][1:]\n",
        "#, excluyendo el idDeportista que ahora es la clave),\n",
        "#y la sigla del país del equipo (l[1][1])."
      ],
      "metadata": {
        "id": "x3OXTOKx9fWp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaPaises = deportistaOlimpicoRDD\\\n",
        "  .map(lambda l: [l[-1],l[:-1]])\\\n",
        "  .join(equiposOlimpicosRDD.map(lambda x: [ x[0],x[2]]))\\\n",
        "  .map(lambda l: (l[1][0][0], (l[0], l[1][0][1:],l[1][1])))\\\n",
        "  .join(resultadoGanador.map(lambda l : [l[2],l[1]]))\\\n"
      ],
      "metadata": {
        "id": "blR-ruZKpcoB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaPaises.takeSample(False,6,25)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J-RCwGrO2fiQ",
        "outputId": "54a155f5-4b0a-406e-f641-a3be786741f0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('54832',\n",
              "  (('982', ['Carlos Jimnez Snchez', '1', '24', '201', '100'], 'ESP'),\n",
              "   'Silver')),\n",
              " ('135219',\n",
              "  (('944', ['Urka olnir', '2', '22', '173', '64'], 'SLO'), 'Bronze')),\n",
              " ('106724',\n",
              "  (('705', ['Adrie Ard Schenk', '1', '19', '190', '90'], 'NED'), 'Gold')),\n",
              " ('112317',\n",
              "  (('1096', ['Justus Ketchum Smith', '1', '26', '0', '0'], 'USA'), 'Gold')),\n",
              " ('124518',\n",
              "  (('705', ['Daniel Daan van Dijk', '1', '21', '0', '0'], 'NED'), 'Gold')),\n",
              " ('42668',\n",
              "  (('1096', ['Joseph Tilford Lee Joe Greene', '1', '25', '183', '70'], 'USA'),\n",
              "   'Bronze'))]"
            ]
          },
          "metadata": {},
          "execution_count": 184
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaPaises.take(6)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uhgA7McU9-ek",
        "outputId": "dd2d7024-6805-4d93-ddbb-388a09ad6d41"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('7597',\n",
              "  (('199', ['Bao Yingying', '2', '24', '172', '67'], 'CHN'), 'Silver')),\n",
              " ('17282', (('199', ['Cai Huijue', '2', '16', '174', '63'], 'CHN'), 'Bronze')),\n",
              " ('17996',\n",
              "  (('199', ['Cao Mianying', '2', '21', '176', '71'], 'CHN'), 'Silver')),\n",
              " ('19779', (('199', ['Chang Si', '2', '25', '170', '56'], 'CHN'), 'Silver')),\n",
              " ('19791',\n",
              "  (('199', ['Chang Yongxiang', '1', '24', '178', '74'], 'CHN'), 'Silver')),\n",
              " ('20215', (('199', ['Chen Jin', '1', '22', '181', '73'], 'CHN'), 'Bronze'))]"
            ]
          },
          "metadata": {},
          "execution_count": 185
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "valoresMedallas = {'Gold':7,\n",
        "                   'Silver': 5,\n",
        "                   'Bronze':4}"
      ],
      "metadata": {
        "id": "y8CDiHZPxe6O"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paisesMedallas = deportistaPaises.map(lambda x: (x[1][0][2], valoresMedallas[x[1][1] ]))"
      ],
      "metadata": {
        "id": "UHriWoJGxejr"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paisesMedallas.take(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xZlGchEL3sqt",
        "outputId": "0e8588d5-8e69-4da2-a58d-6dfa9d50528c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('CHN', 5), ('CHN', 4), ('CHN', 5), ('CHN', 5)]"
            ]
          },
          "metadata": {},
          "execution_count": 188
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from operator import add\n",
        "conclusion = paisesMedallas.reduceByKey((add))\\\n",
        "                           .sortBy(lambda x: x[1],ascending=False)"
      ],
      "metadata": {
        "id": "zEwe8m1xx4-X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "conclusion.take(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LFn-J2lSx45L",
        "outputId": "5c428e8e-738b-4398-b8a9-ae058fa5b7cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('USA', 32137),\n",
              " ('URS', 14834),\n",
              " ('GBR', 10925),\n",
              " ('GER', 10896),\n",
              " ('FRA', 9265),\n",
              " ('ITA', 8755),\n",
              " ('SWE', 8110),\n",
              " ('CAN', 7209),\n",
              " ('AUS', 6755),\n",
              " ('HUN', 6142)]"
            ]
          },
          "metadata": {},
          "execution_count": 190
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#sc.stop()"
      ],
      "metadata": {
        "id": "_5Y6pFCIAg-v"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Dataframes"
      ],
      "metadata": {
        "id": "WNK96Orh_mCr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.types import StructType,StructField\n",
        "from pyspark.sql.types import IntegerType, StringType, FloatType\n",
        "from pyspark.sql.types import Row\n",
        "from pyspark.sql import SQLContext"
      ],
      "metadata": {
        "id": "cxses-sCx40g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sc = SparkContext(master=\"local\",appName=\"Dataframes\")\n",
        "sqlContext = SQLContext(spark)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 200
        },
        "id": "hSNt25Lqx4kE",
        "outputId": "7c136bde-e9b8-4a9a-96c5-1df46317af0b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "ValueError",
          "evalue": "Cannot run multiple SparkContexts at once; existing SparkContext(app=Dataframes, master=local) created by __init__ at <ipython-input-142-56654d120c2a>:1 ",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mValueError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-147-56654d120c2a>\u001b[0m in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0msc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSparkContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmaster\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"local\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mappName\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"Dataframes\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0msqlContext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mSQLContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mspark\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/content/spark-3.5.1-bin-hadoop3/python/pyspark/context.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls, udf_profiler_cls, memory_profiler_cls)\u001b[0m\n\u001b[1;32m    199\u001b[0m             )\n\u001b[1;32m    200\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 201\u001b[0;31m         \u001b[0mSparkContext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_ensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgateway\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgateway\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconf\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mconf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    202\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    203\u001b[0m             self._do_init(\n",
            "\u001b[0;32m/content/spark-3.5.1-bin-hadoop3/python/pyspark/context.py\u001b[0m in \u001b[0;36m_ensure_initialized\u001b[0;34m(cls, instance, gateway, conf)\u001b[0m\n\u001b[1;32m    447\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    448\u001b[0m                     \u001b[0;31m# Raise error if there is already a running Spark context\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 449\u001b[0;31m                     raise ValueError(\n\u001b[0m\u001b[1;32m    450\u001b[0m                         \u001b[0;34m\"Cannot run multiple SparkContexts at once; \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    451\u001b[0m                         \u001b[0;34m\"existing SparkContext(app=%s, master=%s)\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mValueError\u001b[0m: Cannot run multiple SparkContexts at once; existing SparkContext(app=Dataframes, master=local) created by __init__ at <ipython-input-142-56654d120c2a>:1 "
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Hacer dataframes"
      ],
      "metadata": {
        "id": "yWS0UGhiGdnz"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "juegoSchema = StructType([\n",
        "    StructField(\"juego_id\", IntegerType(),False),#Nombre, tipo,  si no se requiere o no\n",
        "    StructField(\"anio\", StringType(),False), # False: Debe estar presente\n",
        "    StructField(\"temporada\",StringType(),False),\n",
        "    StructField(\"ciudad\",StringType(),False)\n",
        "])\n"
      ],
      "metadata": {
        "id": "jEErCixFAzue"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "juegoDF = sqlContext.read.schema(juegoSchema)\\\n",
        "  .option(\"header\",\"true\").csv(path+\"juegos.csv\")"
      ],
      "metadata": {
        "id": "zRaP8M6LAzob"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "juegoDF.show(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hcMyYWSBAzhM",
        "outputId": "b728b1b1-0afe-48bb-f8a2-3aabb9ee11ad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------+-----------+---------+------+\n",
            "|juego_id|       anio|temporada|ciudad|\n",
            "+--------+-----------+---------+------+\n",
            "|       1|1896 Verano|     1896|Verano|\n",
            "|       2|1900 Verano|     1900|Verano|\n",
            "|       3|1904 Verano|     1904|Verano|\n",
            "|       4|1906 Verano|     1906|Verano|\n",
            "+--------+-----------+---------+------+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "spark # se pueden encontar los sc no cerrados"
      ],
      "metadata": {
        "id": "qfHeV1oiAzbR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def removeHeader(index,iterator):\n",
        "    return iter(list(iterator)[1:])"
      ],
      "metadata": {
        "id": "uiMM8U8VAzUY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD = deportistaOlimpicoRDD.mapPartitionsWithIndex(removeHeader)"
      ],
      "metadata": {
        "id": "jLv-9PoAAzOU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD.take(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Wc3COnWxAzHG",
        "outputId": "c8b6a702-6532-4881-8b0d-191d3ba84705"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1', 'A Dijiang', '1', '24', '180', '80', '199'],\n",
              " ['2', 'A Lamusi', '1', '23', '170', '60', '199'],\n",
              " ['3', 'Gunnar Nielsen Aaby', '1', '24', '0', '0', '273'],\n",
              " ['4', 'Edgar Lindenau Aabye', '1', '34', '0', '0', '278'],\n",
              " ['5', 'Christine Jacoba Aaftink', '2', '21', '185', '82', '705']]"
            ]
          },
          "metadata": {},
          "execution_count": 192
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaOlimpicoRDD = deportistaOlimpicoRDD.map(lambda l: (\n",
        "    int(l[0]),\n",
        "    l[1],\n",
        "    int(l[2]),\n",
        "    int(l[3]),\n",
        "    int(l[4]),\n",
        "    float(l[5]),\n",
        "    int(l[6]),\n",
        "\n",
        "))"
      ],
      "metadata": {
        "id": "maGL3u-aAy_R"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "schema = StructType([\n",
        "    StructField(\"deportista_id\",IntegerType(),False),\n",
        "    StructField(\"nombre\",StringType(),False),\n",
        "    StructField(\"genero\",IntegerType(),False),\n",
        "    StructField(\"edad\",IntegerType(),False),\n",
        "    StructField(\"altura\",IntegerType(),False),\n",
        "    StructField(\"peso\",FloatType(),False),\n",
        "    StructField(\"equipo_id\",IntegerType(),False),\n",
        "])"
      ],
      "metadata": {
        "id": "_hFazpc4AyxN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF = sqlContext.createDataFrame(deportistaOlimpicoRDD,schema)"
      ],
      "metadata": {
        "id": "TIlvOnC5D2vt"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eFc8xMiVImrS",
        "outputId": "46d55bf6-5393-4339-eec1-89b724a3fb80"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "|deportista_id|              nombre|genero|edad|altura|peso|equipo_id|\n",
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "|            1|           A Dijiang|     1|  24|   180|80.0|      199|\n",
            "|            2|            A Lamusi|     1|  23|   170|60.0|      199|\n",
            "|            3| Gunnar Nielsen Aaby|     1|  24|     0| 0.0|      273|\n",
            "|            4|Edgar Lindenau Aabye|     1|  34|     0| 0.0|      278|\n",
            "|            5|Christine Jacoba ...|     2|  21|   185|82.0|      705|\n",
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eventoSchema = StructType([\n",
        "    StructField(\"evento_id\", IntegerType(), False),\n",
        "    StructField(\"evento\", StringType(), False),\n",
        "    StructField(\"deporte_id\", IntegerType(), False)\n",
        "])\n",
        "\n",
        "eventoDF = sqlContext.read.format(\"csv\").\\\n",
        "        option(\"header\", True).\\\n",
        "        schema(eventoSchema).\\\n",
        "        load(path+\"evento.csv\")"
      ],
      "metadata": {
        "id": "WfmlN9ChD2oJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "eventoDF.show(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jnRIEUboFHID",
        "outputId": "6077733b-2ff9-49b8-9642-65d595f4cf94"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+--------------------+----------+\n",
            "|evento_id|              evento|deporte_id|\n",
            "+---------+--------------------+----------+\n",
            "|        1|Basketball Men's ...|         1|\n",
            "|        2|Judo Men's Extra-...|         2|\n",
            "|        3|Football Men's Fo...|         3|\n",
            "|        4|Tug-Of-War Men's ...|         4|\n",
            "+---------+--------------------+----------+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "paisesSchema = StructType([\n",
        "    StructField(\"paises_id\", IntegerType(), False),\n",
        "    StructField(\"equipo\", StringType(), False),\n",
        "    StructField(\"sigla\", StringType(), False)\n",
        "])\n",
        "\n",
        "paisesDF = sqlContext.read.format(\"csv\").\\\n",
        "        option(\"header\", True).\\\n",
        "        schema(paisesSchema).\\\n",
        "        load(path+\"paises.csv\")"
      ],
      "metadata": {
        "id": "WSygFyekD2eZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paisesDF.show(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aK18aJJED2VO",
        "outputId": "8260d007-77e9-4d98-d778-9493cd626bad"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+--------------------+-----+\n",
            "|paises_id|              equipo|sigla|\n",
            "+---------+--------------------+-----+\n",
            "|        1|         30. Februar|  AUT|\n",
            "|        2|A North American ...|  MEX|\n",
            "|        3|           Acipactli|  MEX|\n",
            "|        4|             Acturus|  ARG|\n",
            "+---------+--------------------+-----+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultadosSchema = StructType([\n",
        "    StructField(\"resultado_id\", IntegerType(), False),\n",
        "    StructField(\"medalla\", StringType(), False),\n",
        "    StructField(\"deportista_id\", IntegerType(), False),\n",
        "    StructField(\"juego_id\", IntegerType(), False),\n",
        "    StructField(\"evento_id\", IntegerType(), False)\n",
        "])\n",
        "\n",
        "resultadosDF = sqlContext.read.format(\"csv\").\\\n",
        "            option(\"header\", True).\\\n",
        "            schema(resultadosSchema).\\\n",
        "            load(path+\"resultados.csv\")"
      ],
      "metadata": {
        "id": "IZaMpjt_D2NB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "resultadosDF.show(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-3drO_GwD2G9",
        "outputId": "fc7097d9-7443-44b3-f4ad-01696444b2a9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-------+-------------+--------+---------+\n",
            "|resultado_id|medalla|deportista_id|juego_id|evento_id|\n",
            "+------------+-------+-------------+--------+---------+\n",
            "|           1|     NA|            1|      39|        1|\n",
            "|           2|     NA|            2|      49|        2|\n",
            "|           3|     NA|            3|       7|        3|\n",
            "|           4|   Gold|            4|       2|        4|\n",
            "+------------+-------+-------------+--------+---------+\n",
            "only showing top 4 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "paisesSchema = StructType([\n",
        "    StructField(\"paises_id\", IntegerType(), False),\n",
        "    StructField(\"equipo\", StringType(), False),\n",
        "    StructField(\"sigla\", StringType(), False)\n",
        "])"
      ],
      "metadata": {
        "id": "_9BtijaRG3SP"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paisesDF = sqlContext.read.format(\"csv\").\\\n",
        "        option(\"header\", True).\\\n",
        "        schema(paisesSchema).\\\n",
        "        load(path+\"paises.csv\")"
      ],
      "metadata": {
        "id": "dfYQx54JHPJc"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "paisesDF.show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lXrLtsOyHqbi",
        "outputId": "aa8ba086-5a2b-4f9c-b2d0-c49ffb973005"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+--------------------+-----+\n",
            "|paises_id|              equipo|sigla|\n",
            "+---------+--------------------+-----+\n",
            "|        1|         30. Februar|  AUT|\n",
            "|        2|A North American ...|  MEX|\n",
            "|        3|           Acipactli|  MEX|\n",
            "|        4|             Acturus|  ARG|\n",
            "|        5|         Afghanistan|  AFG|\n",
            "+---------+--------------------+-----+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportesSchema = StructType([\n",
        "    StructField(\"deporte_id\", IntegerType(), False),\n",
        "    StructField(\"deporte\", StringType(), False),\n",
        "])\n",
        "deportesDF = sqlContext.read.format(\"csv\").\\\n",
        "        option(\"header\", True).\\\n",
        "        schema(deportesSchema).\\\n",
        "        load(path+\"deporte.csv\")"
      ],
      "metadata": {
        "id": "jev-YY56D2BR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportesDF.show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "91U_hJi8HtOU",
        "outputId": "2ea71c46-b61c-4faf-ba62-7baea3e11114"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+-------------+\n",
            "|deporte_id|      deporte|\n",
            "+----------+-------------+\n",
            "|         1|   Basketball|\n",
            "|         2|         Judo|\n",
            "|         3|     Football|\n",
            "|         4|   Tug-Of-War|\n",
            "|         5|Speed Skating|\n",
            "+----------+-------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportesDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CwpGepchD17S",
        "outputId": "00686e69-e9bd-442b-977e-9889e37a1c06"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- deporte_id: integer (nullable = true)\n",
            " |-- deporte: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J5XVd_BDD1xY",
        "outputId": "2c932231-5d32-4cbd-8cb9-8a3f213d4924"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- deportista_id: integer (nullable = false)\n",
            " |-- nombre: string (nullable = false)\n",
            " |-- genero: integer (nullable = false)\n",
            " |-- edad: integer (nullable = false)\n",
            " |-- altura: integer (nullable = false)\n",
            " |-- peso: float (nullable = false)\n",
            " |-- equipo_id: integer (nullable = false)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF = deportistaDF.withColumnRenamed(\"genero\",\"sexo\").drop(\"altura\")"
      ],
      "metadata": {
        "id": "QQUNiFXjD1oj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KYJqK_DKD1YR",
        "outputId": "2291b181-15a0-4492-f440-0e653825a486"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- deportista_id: integer (nullable = false)\n",
            " |-- nombre: string (nullable = false)\n",
            " |-- sexo: integer (nullable = false)\n",
            " |-- edad: integer (nullable = false)\n",
            " |-- peso: float (nullable = false)\n",
            " |-- equipo_id: integer (nullable = false)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import *\n",
        "deportistaDF = deportistaDF.select(\"deportista_id\",\"nombre\",\n",
        "                    col(\"edad\").alias(\"edadAlJugar\"),\n",
        "                    \"equipo_id\")"
      ],
      "metadata": {
        "id": "8asVz1weJ0Fg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FfUvtpiNJz3G",
        "outputId": "8b6596f3-2d86-4381-9ed3-30a96937dc07"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+--------------------+-----------+---------+\n",
            "|deportista_id|              nombre|edadAlJugar|equipo_id|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "|            1|           A Dijiang|         24|      199|\n",
            "|            2|            A Lamusi|         23|      199|\n",
            "|            3| Gunnar Nielsen Aaby|         24|      273|\n",
            "|            4|Edgar Lindenau Aabye|         34|      278|\n",
            "|            5|Christine Jacoba ...|         21|      705|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF=deportistaDF.filter((deportistaDF.edadAlJugar !=0))"
      ],
      "metadata": {
        "id": "_b62iBHnKfbS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.sort(\"edadAlJugar\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0uURFluyKfSg",
        "outputId": "284ca4b9-8ffb-40e7-e3ca-36b372662aa8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+--------------------+-----------+---------+\n",
            "|deportista_id|              nombre|edadAlJugar|equipo_id|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "|        71691|  Dimitrios Loundras|         10|      333|\n",
            "|        22411|Magdalena Cecilia...|         11|      413|\n",
            "|        70616|          Liu Luyang|         11|      199|\n",
            "|        37333|Carlos Bienvenido...|         11|      982|\n",
            "|        76675|   Marcelle Matthews|         11|      967|\n",
            "|        40129|    Luigina Giavotti|         11|      507|\n",
            "|       118925|Megan Olwen Deven...|         11|      413|\n",
            "|        47618|Sonja Henie Toppi...|         11|      742|\n",
            "|       126307|        Liana Vicens|         11|      825|\n",
            "|        51268|      Beatrice Hutiu|         11|      861|\n",
            "|        52070|        Etsuko Inada|         11|      514|\n",
            "|        72854|      Licia Macchini|         12|      507|\n",
            "|         5291|Marcia Arriaga La...|         12|      656|\n",
            "|        74712|     Carla Marangoni|         12|      507|\n",
            "|        24191| Philippe Cuelenaere|         12|       96|\n",
            "|        74755|Luciana Marcellin...|         12|      507|\n",
            "|        25877|Olga Lucia de Ang...|         12|      225|\n",
            "|        84913|      Clstine N Drin|         12|      237|\n",
            "|        31203|Patricia Anne Pat...|         12|      967|\n",
            "|        96664|Dorothy PoyntonHi...|         12|     1096|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "paisesDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "azO4kkg2RGnU",
        "outputId": "07b57376-49bc-4936-af42-1b48d1e2a04a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- paises_id: integer (nullable = true)\n",
            " |-- equipo: string (nullable = true)\n",
            " |-- sigla: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "75G8fT2VKfLI",
        "outputId": "08c548a3-e841-4719-ec25-7c66a75824b1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- deportista_id: integer (nullable = false)\n",
            " |-- nombre: string (nullable = false)\n",
            " |-- edadAlJugar: integer (nullable = false)\n",
            " |-- equipo_id: integer (nullable = false)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultadosDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-bpjd71bKfFe",
        "outputId": "efafa4aa-88b1-409b-8cb4-510a7e95f35d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- resultado_id: integer (nullable = true)\n",
            " |-- medalla: string (nullable = true)\n",
            " |-- deportista_id: integer (nullable = true)\n",
            " |-- juego_id: integer (nullable = true)\n",
            " |-- evento_id: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "juegoDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RXE92fJMKe_l",
        "outputId": "fc555f72-a976-499d-abeb-fd358eb9d64f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- juego_id: integer (nullable = true)\n",
            " |-- anio: string (nullable = true)\n",
            " |-- temporada: string (nullable = true)\n",
            " |-- ciudad: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportesDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6xaO2KQoKe5g",
        "outputId": "c6b7b377-2c7c-4ef2-aaa6-8020f4298919"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- deporte_id: integer (nullable = true)\n",
            " |-- deporte: string (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "eventoDF.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HjzGBEwEOE6B",
        "outputId": "812db19e-8e95-4b03-827d-a4b7b35677c6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- evento_id: integer (nullable = true)\n",
            " |-- evento: string (nullable = true)\n",
            " |-- deporte_id: integer (nullable = true)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaDF.join(\n",
        "    resultadosDF, deportistaDF.deportista_id == resultadosDF.deportista_id,\"left\")\\\n",
        ".join(juegoDF,juegoDF.juego_id == resultadosDF.juego_id, \"left\")\\\n",
        ".join(eventoDF, eventoDF.evento_id == resultadosDF.evento_id, \"left\")\\\n",
        ".select(deportistaDF.nombre, col(\"edadAlJugar\").alias(\"Edad al jugar\"),\n",
        "        \"medalla\", col(\"anio\").alias(\"Año de juego\"),\n",
        "        eventoDF.evento.alias(\"Nombre de disciplina\")).show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iXIIf4ylKez6",
        "outputId": "765ccc44-9456-4f47-b5ae-02acb45e7967"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+--------------------+-------------+-------+-------------+--------------------+\n",
            "|              nombre|Edad al jugar|medalla| Año de juego|Nombre de disciplina|\n",
            "+--------------------+-------------+-------+-------------+--------------------+\n",
            "|           A Dijiang|           24|     NA|  1992 Verano|Basketball Men's ...|\n",
            "|            A Lamusi|           23|     NA|  2012 Verano|Judo Men's Extra-...|\n",
            "| Gunnar Nielsen Aaby|           24|     NA|  1920 Verano|Football Men's Fo...|\n",
            "|Edgar Lindenau Aabye|           34|   Gold|  1900 Verano|Tug-Of-War Men's ...|\n",
            "|Christine Jacoba ...|           21|     NA|1994 Invierno|Speed Skating Wom...|\n",
            "+--------------------+-------------+-------+-------------+--------------------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultadosDF.filter(resultadosDF.medalla != \"NA\")\\\n",
        "  .join(deportistaDF, deportistaDF.deportista_id == resultadosDF.deportista_id,\"left\")\\\n",
        "  .join(paisesDF, paisesDF.paises_id== deportistaDF.equipo_id, \"left\")\\\n",
        "  .select(\"medalla\",\"equipo\",\"sigla\")\\\n",
        "  .sort( col(\"sigla\").desc()).show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1LE2H8w9KesY",
        "outputId": "90568fcd-2156-4720-b096-d5a25f1fd342"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------+-----+\n",
            "|medalla|  equipo|sigla|\n",
            "+-------+--------+-----+\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "+-------+--------+-----+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio = deportistaDF \\\n",
        "    .join(\n",
        "        resultadosDF,\n",
        "        deportistaDF.deportista_id == resultadosDF.deportista_id,\n",
        "        \"left\"\n",
        "    ) \\\n",
        "    .join(\n",
        "        juegoDF,\n",
        "        juegoDF.juego_id == resultadosDF.juego_id,\n",
        "        \"left\"\n",
        "    ) \\\n",
        "    .join(\n",
        "        paisesDF,\n",
        "        deportistaDF.equipo_id == paisesDF.paises_id,\n",
        "        \"left\"\n",
        "    ) \\\n",
        "    .join(\n",
        "        eventoDF,\n",
        "        eventoDF.evento_id == resultadosDF.evento_id,\n",
        "        \"left\"\n",
        "    ) \\\n",
        "    .join(\n",
        "        deportesDF,\n",
        "        eventoDF.deporte_id == deportesDF.deporte_id,\n",
        "        \"left\"\n",
        "    ) \\\n",
        "    .select(\n",
        "        \"sigla\",\n",
        "        col(\"anio\").alias(\"Año de competencia\"),\n",
        "        \"medalla\",\n",
        "        eventoDF.evento.alias(\"Nombre subdisciplina\"),\n",
        "        deportesDF.deporte.alias(\"Nombre disciplina\"),\n",
        "        deportistaDF.nombre\n",
        "    )\\\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZAJnkODuKej9",
        "outputId": "e015a687-154e-42a7-ae65-ad56ced14e84"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+------------------+-------+--------------------+--------------------+--------------------+\n",
            "|sigla|Año de competencia|medalla|Nombre subdisciplina|   Nombre disciplina|              nombre|\n",
            "+-----+------------------+-------+--------------------+--------------------+--------------------+\n",
            "|  CHN|       1992 Verano|     NA|Basketball Men's ...|          Basketball|           A Dijiang|\n",
            "|  CHN|       2012 Verano|     NA|Judo Men's Extra-...|                Judo|            A Lamusi|\n",
            "|  DEN|       1920 Verano|     NA|Football Men's Fo...|            Football| Gunnar Nielsen Aaby|\n",
            "|  SWE|       1900 Verano|   Gold|Tug-Of-War Men's ...|          Tug-Of-War|Edgar Lindenau Aabye|\n",
            "|  NED|     1994 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  NED|     1994 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  NED|     1992 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  NED|     1992 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  NED|     1988 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  NED|     1988 Invierno|     NA|Speed Skating Wom...|       Speed Skating|Christine Jacoba ...|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1992 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1992 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1992 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1992 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|     Per Knut Aaland|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|        John Aalberg|\n",
            "|  USA|     1994 Invierno|     NA|Cross Country Ski...|Cross Country Skiing|        John Aalberg|\n",
            "+-----+------------------+-------+--------------------+--------------------+--------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.show()"
      ],
      "metadata": {
        "id": "ClOG2mNCKecp"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio2=medallistaXAnio.filter(medallistaXAnio.medalla != \"NA\")\\\n",
        ".sort(\"Año de competencia\")\\\n",
        ".groupBy(\"sigla\",\"Año de competencia\", \"Nombre subdisciplina\")\\\n",
        ".count()"
      ],
      "metadata": {
        "id": "72JCJ7j2UGAE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio2.printSchema()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "08KSUdffUF5y",
        "outputId": "dbff2b21-1212-4b58-c8a8-9aaa06c76d41"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "root\n",
            " |-- sigla: string (nullable = true)\n",
            " |-- Año de competencia: string (nullable = true)\n",
            " |-- Nombre subdisciplina: string (nullable = true)\n",
            " |-- count: long (nullable = false)\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio2.groupBy(\"sigla\",\"Año de competencia\")\\\n",
        "  .agg(sum(\"count\").alias(\"Total de medallas\"),\\\n",
        "  avg(\"count\").alias(\"Medallas promedio\")).show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YK7rLX6IUFya",
        "outputId": "6c364ff0-da62-4278-8df5-13740950ccf9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-----+------------------+-----------------+------------------+\n",
            "|sigla|Año de competencia|Total de medallas| Medallas promedio|\n",
            "+-----+------------------+-----------------+------------------+\n",
            "|  SWE|       1976 Verano|               10|               2.0|\n",
            "|  MGL|       2008 Verano|                5|              1.25|\n",
            "|  SUI|     2014 Invierno|               29|3.2222222222222223|\n",
            "|  ETH|       2004 Verano|                7|              1.75|\n",
            "|  BEL|       2000 Verano|                7|               1.4|\n",
            "|  AUT|       1928 Verano|                5|              1.25|\n",
            "|  SYR|       1984 Verano|                1|               1.0|\n",
            "|  NED|     1992 Invierno|                4|1.3333333333333333|\n",
            "|  MAS|       2012 Verano|                2|               1.0|\n",
            "|  ITA|       1996 Verano|               69| 2.225806451612903|\n",
            "|  THA|       2008 Verano|                4|               1.0|\n",
            "|  URS|     1984 Invierno|               56|               2.8|\n",
            "|  DEN|       1896 Verano|                6|               1.0|\n",
            "|  GRN|       2016 Verano|                1|               1.0|\n",
            "|  DEN|       1956 Verano|                6|               1.5|\n",
            "|  ARG|       2004 Verano|               49| 8.166666666666666|\n",
            "|  CHI|       1988 Verano|                1|               1.0|\n",
            "|  ITA|       1972 Verano|               25|            1.5625|\n",
            "|  GER|       1912 Verano|               53|2.5238095238095237|\n",
            "|  DEN|       1932 Verano|               11|             1.375|\n",
            "+-----+------------------+-----------------+------------------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "SQL"
      ],
      "metadata": {
        "id": "bZuxLqtvW4dL"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#resultadosDF.registerTempTable(\"resultado\")\n",
        "#deportistaDF.registerTempTable(\"deportista\")\n",
        "#paisesDF.registerTempTable(\"paises\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8SqGRPbqUFr_",
        "outputId": "13c1e5b7-9ea7-4ff1-ab80-4fa0cd5f1540"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "/content/spark-3.5.1-bin-hadoop3/python/pyspark/sql/dataframe.py:329: FutureWarning: Deprecated in 2.0, use createOrReplaceTempView instead.\n",
            "  warnings.warn(\"Deprecated in 2.0, use createOrReplaceTempView instead.\", FutureWarning)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "resultadosDF.createOrReplaceTempView (\"resultado\")\n",
        "deportistaDF.createOrReplaceTempView (\"deportista\")\n",
        "paisesDF.createOrReplaceTempView (\"paises\")"
      ],
      "metadata": {
        "id": "1p-TwdliXOlJ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "sqlContext.sql(\"SELECT * FROM deportista\").show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ftt6iRcFUFjp",
        "outputId": "863acbc0-18f0-4bb9-c0f8-26b2b9c1a1ca"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+--------------------+-----------+---------+\n",
            "|deportista_id|              nombre|edadAlJugar|equipo_id|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "|            1|           A Dijiang|         24|      199|\n",
            "|            2|            A Lamusi|         23|      199|\n",
            "|            3| Gunnar Nielsen Aaby|         24|      273|\n",
            "|            4|Edgar Lindenau Aabye|         34|      278|\n",
            "|            5|Christine Jacoba ...|         21|      705|\n",
            "+-------------+--------------------+-----------+---------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sqlContext.sql(\"SELECT * FROM resultado\").show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1aCiR7naYGz-",
        "outputId": "6763dc73-09c9-4d9a-c7d6-ae6df66aaf0a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+------------+-------+-------------+--------+---------+\n",
            "|resultado_id|medalla|deportista_id|juego_id|evento_id|\n",
            "+------------+-------+-------------+--------+---------+\n",
            "|           1|     NA|            1|      39|        1|\n",
            "|           2|     NA|            2|      49|        2|\n",
            "|           3|     NA|            3|       7|        3|\n",
            "|           4|   Gold|            4|       2|        4|\n",
            "|           5|     NA|            5|      36|        5|\n",
            "+------------+-------+-------------+--------+---------+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sqlContext.sql(\"SELECT * FROM paises\").show(5)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ZK573d0YKEr",
        "outputId": "917113d6-01f0-4a4f-d875-1fdafcb90797"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+---------+--------------------+-----+\n",
            "|paises_id|              equipo|sigla|\n",
            "+---------+--------------------+-----+\n",
            "|        1|         30. Februar|  AUT|\n",
            "|        2|A North American ...|  MEX|\n",
            "|        3|           Acipactli|  MEX|\n",
            "|        4|             Acturus|  ARG|\n",
            "|        5|         Afghanistan|  AFG|\n",
            "+---------+--------------------+-----+\n",
            "only showing top 5 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "sqlContext.sql(\"\"\"\n",
        "  SELECT medalla, equipo, sigla FROM resultado r\n",
        "  JOIN deportista d\n",
        "  ON r.deportista_id = d.deportista_id\n",
        "  JOIN paises p\n",
        "  ON p.paises_id=d.equipo_id\n",
        "  WHERE medalla <> \"NA\"\n",
        "  ORDER BY sigla DESC\n",
        "\"\"\").show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EDE4b4rmUFdd",
        "outputId": "6dd2d129-d88f-4593-9169-5c2b4445c59b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------+--------+-----+\n",
            "|medalla|  equipo|sigla|\n",
            "+-------+--------+-----+\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "| Silver|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "| Silver|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "| Silver|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "| Bronze|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "| Silver|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "|   Gold|Zimbabwe|  ZIM|\n",
            "+-------+--------+-----+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### UDF"
      ],
      "metadata": {
        "id": "_ekxdFcuZMfJ"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaError = sc.textFile(path+\"deportistaError.csv\")\\\n",
        "  .map( lambda l: l.split(\",\"))"
      ],
      "metadata": {
        "id": "dqlgZbQFKeVh"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaError=deportistaError.mapPartitionsWithIndex(removeHeader)"
      ],
      "metadata": {
        "id": "LiTadechKeNV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaError.take(2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p_Drz4RaD1GI",
        "outputId": "315b16d7-7aef-47df-cf2a-e0c1f13a5b66"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['1', 'A Dijiang', '1', '24', '180', '80', '199'],\n",
              " ['2', 'A Lamusi', '1', '23', '170', '60', '199']]"
            ]
          },
          "metadata": {},
          "execution_count": 300
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaError.map(lambda l: (\n",
        "                    l[0],\n",
        "                    l[1],\n",
        "                    l[2],\n",
        "                    l[3],\n",
        "                    l[4],\n",
        "                    l[5],\n",
        "                    l[6],\n",
        "                      )\n",
        "                    )"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SXauOWafaDtM",
        "outputId": "600aa980-2d59-42f1-e829-5d6006335003"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "PythonRDD[384] at RDD at PythonRDD.scala:53"
            ]
          },
          "metadata": {},
          "execution_count": 302
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaErrorSchema = StructType([\n",
        "    StructField(\"deportista_id\", StringType(), False),\n",
        "    StructField(\"nombre\", StringType(), False),\n",
        "    StructField(\"genero\", StringType(), False),\n",
        "    StructField(\"edad\", StringType(), False),\n",
        "    StructField(\"altura\", StringType(), False),\n",
        "    StructField(\"peso\", StringType(), False),\n",
        "    StructField(\"equipo_id\", StringType(), False)\n",
        "])\n",
        "deportistaErrorDF = sqlContext.createDataFrame(deportistaError, deportistaErrorSchema)"
      ],
      "metadata": {
        "id": "WCq875ADaDon"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaErrorDF.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2m2qFSf0aDjk",
        "outputId": "d76349db-750a-4abd-f220-efd562c45384"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "|deportista_id|              nombre|genero|edad|altura|peso|equipo_id|\n",
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "|            1|           A Dijiang|     1|  24|   180|  80|      199|\n",
            "|            2|            A Lamusi|     1|  23|   170|  60|      199|\n",
            "|            3| Gunnar Nielsen Aaby|     1|  24|      |    |      273|\n",
            "|            4|Edgar Lindenau Aabye|     1|  34|      |    |      278|\n",
            "|            5|Christine Jacoba ...|     2|  21|   185|  82|      705|\n",
            "|            6|     Per Knut Aaland|     1|  31|   188|  75|     1096|\n",
            "|            7|        John Aalberg|     1|  31|   183|  72|     1096|\n",
            "|            8|\"Cornelia \"\"Cor\"\"...|     2|  18|   168|    |      705|\n",
            "|            9|    Antti Sami Aalto|     1|  26|   186|  96|      350|\n",
            "|           10|\"Einar Ferdinand ...|     1|  26|      |    |      350|\n",
            "|           11|  Jorma Ilmari Aalto|     1|  22|   182|76.5|      350|\n",
            "|           12|   Jyri Tapani Aalto|     1|  31|   172|  70|      350|\n",
            "|           13|  Minna Maarit Aalto|     2|  30|   159|55.5|      350|\n",
            "|           14|Pirjo Hannele Aal...|     2|  32|   171|  65|      350|\n",
            "|           15|Arvo Ossian Aaltonen|     1|  22|      |    |      350|\n",
            "|           16|Juhamatti Tapio A...|     1|  28|   184|  85|      350|\n",
            "|           17|Paavo Johannes Aa...|     1|  28|   175|  64|      350|\n",
            "|           18|Timo Antero Aaltonen|     1|  31|   189| 130|      350|\n",
            "|           19|Win Valdemar Aalt...|     1|  54|      |    |      350|\n",
            "|           20|  Kjetil Andr Aamodt|     1|  20|   176|  85|      742|\n",
            "+-------------+--------------------+------+----+------+----+---------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql.functions import udf\n",
        "def conversionIntegers(value):\n",
        "  return int(value) if len(value)> 0 else None\n",
        "conversionEnteros_udf = udf(lambda z: conversionIntegers(z), IntegerType())\n",
        "sqlContext.udf.register(\"conversionEnteros_udf\", conversionEnteros_udf)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gzvW7U5DaDe6",
        "outputId": "665e7797-7f72-4ec7-cb94-373a144a2cbe"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<pyspark.sql.udf.UserDefinedFunction at 0x7e2752a4a3b0>"
            ]
          },
          "metadata": {},
          "execution_count": 306
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "deportistaErrorDF.select(conversionEnteros_udf(\"altura\")\\\n",
        "                         .alias(\"Altura udf\")).show()\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BpYjQ4hzaDaC",
        "outputId": "461855a2-f27d-4a87-985e-db61bef64a37"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "+----------+\n",
            "|Altura udf|\n",
            "+----------+\n",
            "|       180|\n",
            "|       170|\n",
            "|      NULL|\n",
            "|      NULL|\n",
            "|       185|\n",
            "|       188|\n",
            "|       183|\n",
            "|       168|\n",
            "|       186|\n",
            "|      NULL|\n",
            "|       182|\n",
            "|       172|\n",
            "|       159|\n",
            "|       171|\n",
            "|      NULL|\n",
            "|       184|\n",
            "|       175|\n",
            "|       189|\n",
            "|      NULL|\n",
            "|       176|\n",
            "+----------+\n",
            "only showing top 20 rows\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Persistencia"
      ],
      "metadata": {
        "id": "dU_DLttrc4Ay"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.storagelevel import StorageLevel"
      ],
      "metadata": {
        "id": "__fWkBMKaDVg"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.is_cached"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xqCjHEyIaDRR",
        "outputId": "88461072-79da-4522-aa2d-361e273eccb6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "False"
            ]
          },
          "metadata": {},
          "execution_count": 309
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.cache()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uVkRfxxTaDM8",
        "outputId": "45ad12ca-cf20-4312-ea89-15810431e537"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MapPartitionsRDD[419] at javaToPython at NativeMethodAccessorImpl.java:0"
            ]
          },
          "metadata": {},
          "execution_count": 310
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.getStorageLevel()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2HDTf35SaDH8",
        "outputId": "1e3ac5fa-006b-4e2f-f08a-3a3647d9756e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "StorageLevel(False, True, False, False, 1)"
            ]
          },
          "metadata": {},
          "execution_count": 311
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.unpersist() # SE quita del disco"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6iXRIlmndkMn",
        "outputId": "f8ff3dbd-87f8-49ee-ff6c-457c774c2338"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MapPartitionsRDD[419] at javaToPython at NativeMethodAccessorImpl.java:0"
            ]
          },
          "metadata": {},
          "execution_count": 317
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.persist(StorageLevel.MEMORY_AND_DISK_2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FLz2S2MGaDDw",
        "outputId": "860d789e-e2e0-464e-c9a1-02feb70ff4d9"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MapPartitionsRDD[419] at javaToPython at NativeMethodAccessorImpl.java:0"
            ]
          },
          "metadata": {},
          "execution_count": 318
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "StorageLevel.MEMORY_AND_DISK_3 = StorageLevel(True,True,False,False,3)\n",
        "#Disco,Memoria,No dice, No dice,Numero de replicaciones"
      ],
      "metadata": {
        "id": "2ZvCHtaSaC_X"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.unpersist() # SE quita del disco"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "auKf71uKaC7A",
        "outputId": "00f7ba3d-b99a-45b5-80c1-00453aad6abf"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MapPartitionsRDD[419] at javaToPython at NativeMethodAccessorImpl.java:0"
            ]
          },
          "metadata": {},
          "execution_count": 321
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "medallistaXAnio.rdd.persist(StorageLevel.MEMORY_AND_DISK_3)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0uXmxxk5aC1W",
        "outputId": "211393ad-bbaa-46bf-a576-93a29ea26bd1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "MapPartitionsRDD[419] at javaToPython at NativeMethodAccessorImpl.java:0"
            ]
          },
          "metadata": {},
          "execution_count": 323
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from pyspark.sql import SparkSession\n",
        "sparkP = SparkSession.builder.appName(\"Particionado\")\\\n",
        "  .master(\"local[5]\").getOrCreate()"
      ],
      "metadata": {
        "id": "wOUy0_SraCwT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "df = sparkP.range(0,20)\n",
        "df.rdd.getNumPartitions()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-bDc5u9jf4a9",
        "outputId": "5d8cbe1c-0ed2-4183-b9a3-defde7decbc2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1"
            ]
          },
          "metadata": {},
          "execution_count": 327
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd1 = sparkP.sparkContext.parallelize((0,20),10)\n",
        "#set dedatos generados, como se quiere la replicacion\n",
        "rdd1.getNumPartitions()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LYSC4fgjf4UR",
        "outputId": "64b60858-1aa1-426f-a62d-795cb13f241e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {},
          "execution_count": 329
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rddDesdeArchivo = sparkP\\\n",
        "  .sparkContext\\\n",
        "  .textFile(path+\"deporte.csv\",10)"
      ],
      "metadata": {
        "id": "XdPzk1e5f4Mq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rddDesdeArchivo.getNumPartitions()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7xlJQnJpf4AG",
        "outputId": "0c810e5c-0ec7-418d-8832-21083f6211b8"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "10"
            ]
          },
          "metadata": {},
          "execution_count": 333
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "path_save ='/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/'"
      ],
      "metadata": {
        "id": "jBMXlnOYhQHb"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rddDesdeArchivo.saveAsTextFile(path_save+\"salidaTexto2\")"
      ],
      "metadata": {
        "id": "clAR_srxaCrB"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!ls /content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d_BT-JCehjqf",
        "outputId": "93f91e64-93bf-47b7-dba9-8c9cf8156860"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "data  salidaTexto  salidaTexto2  Spark\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!ls /content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8owRBamCj_G9",
        "outputId": "e42295ae-3b14-4029-949a-05dd65af2fd2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "part-00000  part-00002\tpart-00004  part-00006\tpart-00008  _SUCCESS\n",
            "part-00001  part-00003\tpart-00005  part-00007\tpart-00009\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "!head -n 5 /content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/part-00004"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "y0VdC7S3kfMy",
        "outputId": "67a9e9f0-3973-4604-ca02-b9bd6c6510dd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "29,Cycling\n",
            "30,Diving\n",
            "31,Canoeing\n",
            "32,Tennis\n",
            "33,Modern Pentathlon\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "rdd = sparkP.sparkContext.wholeTextFiles(\"/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/*\")\n",
        "#* es una muletilla, unawildcard quepermite cargar todos los archivos que existen dentro de la carpeta"
      ],
      "metadata": {
        "id": "QcbNiZS-k7-d"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd.take(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tX6wLIolmBg2",
        "outputId": "37eba138-ddff-46ac-f6ff-47a4d0640556"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('file:/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/part-00000',\n",
              "  'deporte_id,deporte\\n1,Basketball\\n2,Judo\\n3,Football\\n4,Tug-Of-War\\n5,Speed Skating\\n6,Cross Country Skiing\\n'),\n",
              " ('file:/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/part-00001',\n",
              "  '7,Athletics\\n8,Ice Hockey\\n9,Swimming\\n10,Badminton\\n11,Sailing\\n12,Biathlon\\n13,Gymnastics\\n14,Art Competitions\\n'),\n",
              " ('file:/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/part-00002',\n",
              "  '15,Alpine Skiing\\n16,Handball\\n17,Weightlifting\\n18,Wrestling\\n19,Luge\\n20,Water Polo\\n'),\n",
              " ('file:/content/drive/MyDrive/EstudiosAparte/Platzi/SPARK/salidaTexto2/part-00003',\n",
              "  '21,Hockey\\n22,Rowing\\n23,Bobsleigh\\n24,Fencing\\n25,Equestrianism\\n26,Shooting\\n27,Boxing\\n28,Taekwondo\\n')]"
            ]
          },
          "metadata": {},
          "execution_count": 371
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "lista = rdd.mapValues(lambda x: x.split()).collect()"
      ],
      "metadata": {
        "id": "FNnad29Glexq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "lista = [l[0] for l in lista]\n",
        "lista.sort()"
      ],
      "metadata": {
        "id": "syRhgeQ1ledk"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd2 = sparkP\\\n",
        "  .sparkContext\\\n",
        "  .textFile(\",\".join(lista),10).map(lambda l:l.split(\",\"))"
      ],
      "metadata": {
        "id": "hC_8BXJwnuud"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "rdd2.take(4)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wObGCkOHoITq",
        "outputId": "03c3a7a9-5410-4e0a-92b4-444beccba761"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[['deporte_id', 'deporte'],\n",
              " ['1', 'Basketball'],\n",
              " ['2', 'Judo'],\n",
              " ['3', 'Football']]"
            ]
          },
          "metadata": {},
          "execution_count": 376
        }
      ]
    }
  ]
}