setup_environment: collect log file url and install packages

open-contracting · Apr 19, 2022 · 5e6259a · 5e6259a
1 parent 30df5a5
commit 5e6259a
Showing 1 changed file with 35 additions and 5 deletions.
diff --git a/setup_environment.ipynb b/setup_environment.ipynb
@@ -5,7 +5,7 @@
     "colab": {
       "name": "setup_notebook",
       "provenance": [],
-      "authorship_tag": "ABX9TyPpPT0FsHcqUr/d/JNLgAmy"
+      "authorship_tag": "ABX9TyPNHL0X/U/czPOZW0el9YnT"
     },
     "kernelspec": {
       "name": "python3",
@@ -44,13 +44,37 @@
       "execution_count": null,
       "outputs": []
     },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "To check that the Kingfisher Collect scraper for your collection completed sucessfully, set the URL of the log file for the job from [Kingfisher Collect](https://collect.kingfisher.open-contracting.org), e.g. `https://collect.kingfisher.open-contracting.org/logs/kingfisher/united_kingdom_contracts_finder_releases/2ef35b40b93b11ec88920c9d92c523cb.log`."
+      ],
+      "metadata": {
+        "id": "CEmXhx6wn_Ft"
+      }
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "log_url = ''\n",
+        "\n",
+        "if log_url != '':\n",
+        "\n",
+        "  scrapy_password = getpass.getpass('Enter the password for collect.kingfisher.open-contracting.org:')"
+      ],
+      "metadata": {
+        "id": "bhtlijjgmBU2"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
     {
       "cell_type": "markdown",
       "metadata": {
         "id": "F8fwq_F6XMLs"
       },
       "source": [
-        "Enter credentials:"
+        "Enter your Kingfisher Process credentials:"
       ]
     },
     {
@@ -61,7 +85,6 @@
       "source": [
         "import getpass\n",
         "\n",
-        "print('Enter your Kingfisher credentials')\n",
         "user = input('Username:')\n",
         "password = getpass.getpass('Password:')"
       ],
@@ -86,7 +109,13 @@
         "# Install Kingfisher Colab and required packages\n",
         "%shell pip install --upgrade 'ocdskingfishercolab<0.4' psycopg2-binary > pip.log\n",
         "\n",
+        "# Install Scrapy Log Analyzer\n",
+        "pip install git+https://github.com/open-contracting/scrapy-log-analyzer.git\n",
+        "\n",
         "# Import libraries and functions\n",
+        "import pandas as pd\n",
+        "import requests\n",
+        "\n",
         "from google.colab.data_table import DataTable\n",
         "from ocdskingfishercolab import (\n",
         "    list_source_ids,\n",
@@ -96,7 +125,8 @@
         "    save_dataframe_to_sheet,\n",
         "    save_dataframe_to_spreadsheet,\n",
         "    set_search_path)\n",
-        "import pandas as pd\n",
+        "from pprint import pprint\n",
+        "from scrapyloganalyzer import ScrapyLogFile\n",
         "\n",
         "# Load https://pypi.org/project/ipython-sql/\n",
         "%load_ext sql \n",
@@ -144,4 +174,4 @@
       "outputs": []
     }
   ]
-}
+}