template_red_flags_checks_registry.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sfg_SbQWBCmW"
      },
      "source": [
        "## Setup\n",
        "\n",
        "*You must run the cells in this section each time you connect to a new runtime. For example, when you return to the notebook after an idle timeout, when the runtime crashes, or when you restart or factory reset the runtime.*"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sowWi_Ve_Lm3"
      },
      "source": [
        "Install requirements (*Note: ocdskingfishercolab installs google-colab, which expects specific versions of pandas and numpy*):\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X4nmyvOa_Ls7"
      },
      "outputs": [],
      "source": [
        "! pip install --upgrade pip > pip.log\n",
        "! pip install --upgrade 'ocdskingfishercolab<0.4' ipywidgets psycopg2-binary >> pip.log"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "tezDNj4uAOHq"
      },
      "outputs": [],
      "source": [
        "# @title Import packages and load extensions { display-mode: \"form\" }\n",
        "\n",
        "import gzip\n",
        "import json\n",
        "import os\n",
        "import shutil\n",
        "import tempfile\n",
        "from collections import Counter\n",
        "from datetime import datetime, timezone\n",
        "from pathlib import Path\n",
        "\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from dateutil.relativedelta import relativedelta\n",
        "from google.colab.data_table import DataTable\n",
        "from google.colab.files import download\n",
        "from ipywidgets import widgets\n",
        "from ocdskingfishercolab import (\n",
        "    authenticate_gspread,\n",
        "    calculate_coverage,\n",
        "    download_dataframe_as_csv,\n",
        "    format_thousands,\n",
        "    render_json,\n",
        "    save_dataframe_to_sheet,\n",
        "    save_dataframe_to_spreadsheet,\n",
        "    set_dark_mode,\n",
        "    set_light_mode,\n",
        "    set_spreadsheet_name,\n",
        ")\n",
        "\n",
        "# Load https://pypi.org/project/ipython-sql/\n",
        "%load_ext sql\n",
        "# Load https://colab.research.google.com/notebooks/data_table.ipynb\n",
        "%load_ext google.colab.data_table"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "--8vgOiP_58f"
      },
      "outputs": [],
      "source": [
        "# @title Configure the notebook environment { display-mode: \"form\" }\n",
        "\n",
        "# Increase max columns so that Pandas DataFrames with many columns are rendered as data tables.\n",
        "DataTable.max_columns = 50\n",
        "# Remove the index from data tables for easier copy-pasting to Google Docs.\n",
        "DataTable.include_index = False\n",
        "\n",
        "# Return Pandas DataFrames instead of regular result sets.\n",
        "%config SqlMagic.autopandas = True\n",
        "# Don't print number of rows affected.\n",
        "%config SqlMagic.feedback = False\n",
        "\n",
        "# If you set Tools > Settings > Site > Theme to dark, uncomment this line.\n",
        "# set_dark_mode()\n",
        "# If you are creating plots to copy-paste into reports, uncomment this line.\n",
        "# set_light_mode()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IhnbdjqU1e6p"
      },
      "source": [
        "## Charts Setup\n",
        "*You must run the cells in this section each time you connect to a new runtime. For example, when you return to the notebook after an idle timeout, when the runtime crashes, or when you restart or factory reset the runtime.*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "elpUvMf61Ym6"
      },
      "outputs": [],
      "source": [
        "! pip install altair pyarrow==14.0.0 >> pip.log"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "P1aenztz1zK3"
      },
      "source": [
        "Import chart packages and define chart functions. The currently available chart functions are:\n",
        "\n",
        "* Release count\n",
        "* Objects per stage\n",
        "* Releases by month\n",
        "* Objects per year\n",
        "* Top buyers\n",
        "* Usability indicators"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Bip37aP917XY"
      },
      "outputs": [],
      "source": [
        "# @title Chart functions { display-mode: \"form\" }\n",
        "import altair as alt\n",
        "\n",
        "\n",
        "class MissingColumnsError(Exception):\n",
        "    def __init__(self, columns):\n",
        "        super().__init__(f\"The source data is missing one or more of these columns: {columns}\")\n",
        "\n",
        "\n",
        "chart_properties = {\n",
        "    \"width\": 600,\n",
        "    \"height\": 350,\n",
        "    \"padding\": 50,\n",
        "    \"title\": alt.TitleParams(text=\"\", subtitle=[\"\"], fontSize=18),\n",
        "}\n",
        "chart_axis = {\n",
        "    \"titleFontSize\": 14,\n",
        "    \"labelFontSize\": 14,\n",
        "    \"labelPadding\": 5,\n",
        "    \"ticks\": False,\n",
        "    \"domain\": False,\n",
        "}\n",
        "\n",
        "\n",
        "def check_columns(columns, data):\n",
        "    # check if input contains the right columns\n",
        "    if not columns.issubset(data.columns):\n",
        "        raise MissingColumnsError(columns)\n",
        "\n",
        "\n",
        "def plot_release_count(release_counts):\n",
        "    check_columns({\"collection_id\", \"release_type\", \"release_count\", \"ocid_count\"}, release_counts)\n",
        "    return (\n",
        "        alt.Chart(release_counts)\n",
        "        .mark_bar()\n",
        "        .encode(\n",
        "            x=alt.X(\n",
        "                \"release_count\",\n",
        "                type=\"ordinal\",\n",
        "                axis=alt.Axis(title=\"release count\", labelAngle=0),\n",
        "            ),\n",
        "            y=alt.Y(\n",
        "                \"ocid_count\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=\"ocid count\", format=\"~s\", tickCount=5),\n",
        "            ),\n",
        "            color=alt.Color(\n",
        "                \"release_type\",\n",
        "                type=\"nominal\",\n",
        "                title=\"release type\",\n",
        "                scale=alt.Scale(range=[\"#D6E100\", \"#FB6045\", \"#23B2A7\", \"#6C75E1\"]),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(\"release_count\", title=\"release count\"),\n",
        "                alt.Tooltip(\"ocid_count\", title=\"ocid count\", format=\"~s\"),\n",
        "                alt.Tooltip(\"release_type\", title=\"release type\"),\n",
        "                alt.Tooltip(\"collection_id\", title=\"collection id\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )\n",
        "\n",
        "\n",
        "def plot_objects_per_stage(objects_per_stage):\n",
        "    check_columns({\"stage\", \"object_count\"}, objects_per_stage)\n",
        "    stages = [\"planning\", \"tender\", \"awards\", \"contracts\", \"implementation\"]\n",
        "    return (\n",
        "        alt.Chart(objects_per_stage)\n",
        "        .mark_bar(fill=\"#d6e100\")\n",
        "        .encode(\n",
        "            x=alt.X(\n",
        "                \"stage\",\n",
        "                type=\"ordinal\",\n",
        "                scale=alt.Scale(domain=stages),\n",
        "                sort=stages,\n",
        "                axis=alt.Axis(title=\"stage\", labelAngle=0),\n",
        "            ),\n",
        "            y=alt.Y(\n",
        "                \"object_count\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=\"number of objects\", format=\"~s\", tickCount=len(stages)),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(\"stage\", title=\"stage\"),\n",
        "                alt.Tooltip(\"object_count\", title=\"number of objects\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )\n",
        "\n",
        "\n",
        "def plot_releases_by_month(release_dates):\n",
        "    check_columns({\"date\", \"collection_id\", \"release_type\", \"release_count\"}, release_dates)\n",
        "    max_rows = 5000\n",
        "    # check if number of rows is more than 5000\n",
        "    if release_dates.shape[0] > max_rows:\n",
        "        alt.data_transformers.disable_max_rows()\n",
        "\n",
        "    # draw chart\n",
        "    return (\n",
        "        alt.Chart(release_dates)\n",
        "        .mark_line(strokeWidth=3)\n",
        "        .encode(\n",
        "            x=alt.X(\"date\", timeUnit=\"yearmonth\", axis=alt.Axis(title=\"year and month\")),\n",
        "            y=alt.Y(\n",
        "                \"release_count\",\n",
        "                type=\"quantitative\",\n",
        "                aggregate=\"sum\",\n",
        "                axis=alt.Axis(title=\"number of releases\", format=\"~s\", tickCount=5),\n",
        "                scale=alt.Scale(zero=False),\n",
        "            ),\n",
        "            color=alt.Color(\n",
        "                \"release_type\",\n",
        "                type=\"nominal\",\n",
        "                scale=alt.Scale(range=[\"#D6E100\", \"#FB6045\", \"#23B2A7\", \"#6C75E1\"]),\n",
        "                legend=alt.Legend(title=\"release type\"),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(\"date\", timeUnit=\"yearmonth\", title=\"date\"),\n",
        "                alt.Tooltip(\"release_count\", aggregate=\"sum\", title=\"number of releases\"),\n",
        "                alt.Tooltip(\"release_type\", title=\"release type\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )\n",
        "\n",
        "\n",
        "def plot_objects_per_year(objects_per_year):\n",
        "    check_columns({\"year\", \"tenders\", \"awards\"}, objects_per_year)\n",
        "    stages = [\"tenders\", \"awards\"]\n",
        "    return (\n",
        "        alt.Chart(objects_per_year)\n",
        "        .transform_fold(stages)\n",
        "        .mark_line(strokeWidth=3)\n",
        "        .encode(\n",
        "            x=alt.X(\n",
        "                \"year\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=\"year\", format=\".0f\", tickCount=objects_per_year.shape[0]),\n",
        "            ),\n",
        "            y=alt.Y(\n",
        "                \"value\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=\"number of objects\", format=\"~s\", tickCount=5),\n",
        "                scale=alt.Scale(zero=False),\n",
        "            ),\n",
        "            color=alt.Color(\n",
        "                \"key\",\n",
        "                type=\"nominal\",\n",
        "                title=\"object type\",\n",
        "                scale=alt.Scale(domain=stages, range=[\"#D6E100\", \"#FB6045\"]),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(\"year\", title=\"year\", type=\"quantitative\"),\n",
        "                alt.Tooltip(\"value\", title=\"number of objects\", type=\"quantitative\"),\n",
        "                alt.Tooltip(\"key\", title=\"object type\", type=\"nominal\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )\n",
        "\n",
        "\n",
        "def plot_top_buyers(buyers):\n",
        "    check_columns({\"name\", \"total_tenders\"}, buyers)\n",
        "    return (\n",
        "        alt.Chart(buyers)\n",
        "        .mark_bar(fill=\"#d6e100\")\n",
        "        .encode(\n",
        "            x=alt.X(\n",
        "                \"total_tenders\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=\"number of tenders\", format=\"~s\", tickCount=5),\n",
        "            ),\n",
        "            y=alt.Y(\n",
        "                \"name\",\n",
        "                type=\"ordinal\",\n",
        "                axis=alt.Axis(title=\"buyer\", labelAngle=0),\n",
        "                sort=alt.SortField(\"total_tenders\", order=\"descending\"),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(\"name\", title=\"buyer\", type=\"nominal\"),\n",
        "                alt.Tooltip(\"total_tenders\", title=\"number of tenders\", type=\"quantitative\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )\n",
        "\n",
        "\n",
        "def plot_usability_indicators(data, lang=\"English\"):\n",
        "    labels = {\n",
        "        \"English\": {\n",
        "            \"nrow\": \"row_number(indicator)\",\n",
        "            \"sort\": \"calculation\",\n",
        "            \"y_sort\": \"indicator\",\n",
        "            \"groupby\": \"Use case\",\n",
        "            \"title\": \"number of indicators\",\n",
        "            \"tooltip_missing\": \"Missing Fields\",\n",
        "        },\n",
        "        \"Spanish\": {\n",
        "            \"nrow\": \"row_number(Indicador)\",\n",
        "            \"sort\": \"¿Se puede calcular?\",\n",
        "            \"y_sort\": \"Indicador\",\n",
        "            \"groupby\": \"Caso de Uso\",\n",
        "            \"title\": \"Número de indicadores\",\n",
        "            \"tooltip_missing\": \"Campos faltantes\",\n",
        "        },\n",
        "    }\n",
        "    return (\n",
        "        alt.Chart(data)\n",
        "        .transform_window(\n",
        "            nrow=labels[lang][\"nrow\"],\n",
        "            frame=[None, None],\n",
        "            sort=[{\"field\": labels[lang][\"sort\"]}],\n",
        "            groupby=[labels[lang][\"groupby\"]],\n",
        "        )\n",
        "        .mark_circle(size=250, opacity=1)\n",
        "        .encode(\n",
        "            x=alt.X(\n",
        "                \"nrow\",\n",
        "                type=\"quantitative\",\n",
        "                axis=alt.Axis(title=[labels[lang][\"title\"], \"\"], orient=\"top\", tickCount=5),\n",
        "            ),\n",
        "            y=alt.Y(\n",
        "                labels[lang][\"groupby\"],\n",
        "                type=\"nominal\",\n",
        "                sort=alt.Sort(field=labels[lang][\"y_sort\"], op=\"count\", order=\"descending\"),\n",
        "            ),\n",
        "            color=alt.Color(\n",
        "                labels[lang][\"sort\"],\n",
        "                type=\"nominal\",\n",
        "                scale=alt.Scale(range=[\"#fb6045\", \"#d6e100\"]),\n",
        "                legend=alt.Legend(title=[labels[lang][\"sort\"]]),\n",
        "            ),\n",
        "            tooltip=[\n",
        "                alt.Tooltip(labels[lang][\"y_sort\"], type=\"nominal\"),\n",
        "                alt.Tooltip(labels[lang][\"groupby\"], type=\"nominal\"),\n",
        "                alt.Tooltip(labels[lang][\"sort\"], type=\"nominal\"),\n",
        "                alt.Tooltip(labels[lang][\"tooltip_missing\"], type=\"nominal\"),\n",
        "            ],\n",
        "        )\n",
        "        .properties(**chart_properties)\n",
        "        .configure_axis(**chart_axis)\n",
        "        .configure_view(strokeWidth=0)\n",
        "    )"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "25lVf3PwsmmV"
      },
      "source": [
        "## Setup Cardinal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6fzwlrSLOF3w"
      },
      "source": [
        "### Install Cardinal"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1H6SWBCTDq5G"
      },
      "source": [
        "This notebook uses [Cardinal](https://cardinal.readthedocs.io/en/latest/), a Rust package to calculate red flags and the coverage of OCDS data."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6syz0fkkEdgj"
      },
      "outputs": [],
      "source": [
        "# @title Install { display-mode: \"form\" }\n",
        "\n",
        "! curl -sSOL https://github.com/open-contracting/cardinal-rs/releases/download/0.0.5/ocdscardinal-0.0.5-linux-64-bit.zip\n",
        "! unzip -oj ocdscardinal-0.0.5-linux-64-bit.zip ocdscardinal-0.0.5-linux-64-bit/ocdscardinal\n",
        "\n",
        "def cardinal_calculate_coverage(file_name):\n",
        "    coverage = !./ocdscardinal coverage $file_name\n",
        "    fields = (\n",
        "        pd.DataFrame.from_dict(json.loads(coverage[0]), orient=\"index\", columns=[\"count\"])\n",
        "        .reset_index()\n",
        "        .rename(columns={\"index\": \"path\"})\n",
        "    )\n",
        "    # Leaves only object members\n",
        "    fields_table = fields[fields.path.str.contains(\"[a-z]$\")].copy()\n",
        "    fields_table[\"path\"] = fields_table[\"path\"].str.replace(r\"[][]|^/\", \"\", regex=True)\n",
        "    return fields_table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vpPltvi1tjEs"
      },
      "source": [
        "## Setup download data from the Data Registry"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CLc-_oJntg93"
      },
      "outputs": [],
      "source": [
        "# @title Data registry functions{ display-mode: \"form\" }\n",
        "import requests\n",
        "\n",
        "DATA_REGISTRY_BASE_URL = \"https://data.open-contracting.org/en/\"\n",
        "PUBLICATIONS_URL = f\"{DATA_REGISTRY_BASE_URL}publications.json\"\n",
        "\n",
        "\n",
        "def get_publications():\n",
        "    publications = requests.get(PUBLICATIONS_URL, timeout=10).json()\n",
        "    for publication in publications:\n",
        "        publication[\"label\"] = f\"{publication['country']} - {publication['title']}\"\n",
        "    return publications\n",
        "\n",
        "\n",
        "def get_publication_select_box():\n",
        "    return widgets.Dropdown(\n",
        "        options=sorted([entry[\"label\"] for entry in get_publications()]),\n",
        "        description=\"Publication:\",\n",
        "        disabled=False,\n",
        "    )\n",
        "\n",
        "\n",
        "def get_available_years(publication):\n",
        "    years = [\"full\"]\n",
        "    if publication[\"date_from\"] and publication[\"date_to\"]:\n",
        "        year_from = int(publication[\"date_from\"][:4])\n",
        "        year_to = int(publication[\"date_to\"][:4])\n",
        "        years.extend(list(range(year_from, year_to + 1)))\n",
        "    return years\n",
        "\n",
        "\n",
        "def get_years_select_box(publication_select_box):\n",
        "    selected_publication = next(\n",
        "        filter(lambda entry: entry[\"label\"] == publication_select_box.value, get_publications())\n",
        "    )\n",
        "    return (\n",
        "        widgets.Dropdown(\n",
        "            options=get_available_years(selected_publication),\n",
        "            description=\"Year:\",\n",
        "            disabled=False,\n",
        "        ),\n",
        "        selected_publication,\n",
        "    )\n",
        "\n",
        "\n",
        "def download_file(selected_publication, selected_year):\n",
        "    file_name = f\"{selected_publication['source_id']}-{selected_year}.jsonl\"\n",
        "    download_url = (\n",
        "        f'{DATA_REGISTRY_BASE_URL}publication/{selected_publication[\"id\"]}/download?name={selected_year}.jsonl.gz'\n",
        "    )\n",
        "    response = requests.get(download_url, timeout=10)\n",
        "    with tempfile.NamedTemporaryFile() as gz_file:\n",
        "        gz_file.write(response.content)\n",
        "        with gzip.open(gz_file.name) as i, Path(file_name).open(\"wb\") as o:\n",
        "            shutil.copyfileobj(i, o)\n",
        "    return file_name"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vpPltvi1tjEs"
      },
      "source": [
        "## Select a publication to download from the [Data Registry](https://data.open-contracting.org/) and generates its field list"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CLc-_oJntg93"
      },
      "outputs": [],
      "source": [
        "# @title Select the publication to download { display-mode: \"form\" }\n",
        "\n",
        "publication_select_box = get_publication_select_box()\n",
        "\n",
        "publication_select_box"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# @title Select year to download or \"full\" to download all the available years { display-mode: \"form\" }\n",
        "\n",
        "selected_year, selected_publication = get_years_select_box(publication_select_box)\n",
        "selected_year"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# @title Download the selected file { display-mode: \"form\" }\n",
        "\n",
        "file_name = download_file(selected_publication, selected_year.value)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "# @title Extract the list of available fields { display-mode: \"form\" }\n",
        "\n",
        "fields_table = cardinal_calculate_coverage(file_name)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s75VrpzLRNkx"
      },
      "source": [
        "## Usability analysis setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pSR2il4VRPmJ"
      },
      "source": [
        "Use this section to setup the functions needed to perform a usability analysis of the dataset, to identify if a publisher has the necessary fields to calculate 71 procurement indicators related to market opportunity (market description, competition, supplier performance), value for money, internal efficiency, public integrity and service delivery.  For an OCDS publisher, it also calculates the proportion of unique procedures for which it is possible to calculate the indicator (coverage).\n",
        "\n",
        "The usability checks includes all the indicators listed on [OCP's use case guide](https://docs.google.com/spreadsheets/d/1j-Y0ktZiOyhZzi-2GSabBCnzx6fF5lv8h1KYwi_Q9GM/edit#gid=1183427361) and the [Indicators to diagnose the performance of a procurement market document](https://docs.google.com/document/d/1vSJk9-qWSTQEx9ZZc7BUhQZMHvTRcyDYVS2sl8HB__k/edit#heading=h.nrnq1ajwwpqe)."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zVnZn4E1kHg1"
      },
      "outputs": [],
      "source": [
        "# @title Usability functions { display-mode: \"form\" }\n",
        "\n",
        "RELEVANT_RULES = {\n",
        "    \"who\": [\n",
        "        \"buyer/id\",\n",
        "        \"buyer/name\",\n",
        "        \"tender/procuringEntity/id\",\n",
        "        \"tender/procuringEntity/name\",\n",
        "    ],\n",
        "    \"bought what\": [\n",
        "        \"tender/items/classification/id\",\n",
        "        \"awards/items/classification/id\",\n",
        "        \"contracts/items/classification/id\",\n",
        "        \"tender/items/classification/description\",\n",
        "        \"awards/items/classification/description\",\n",
        "        \"contracts/items/classification/description\",\n",
        "        \"tender/items/description\",\n",
        "        \"awards/items/description\",\n",
        "        \"contracts/items/description\",\n",
        "        \"tender/description\",\n",
        "        \"awards/description\",\n",
        "        \"contracts/description\",\n",
        "        \"tender/title\",\n",
        "        \"awards/title\",\n",
        "        \"contracts/title\",\n",
        "    ],\n",
        "    \"from whom\": [\n",
        "        \"awards/suppliers/id\",\n",
        "        \"awards/suppliers/name\",\n",
        "    ],\n",
        "    \"for how much\": [\n",
        "        \"awards/value/amount\",\n",
        "        \"contracts/value/amount\",\n",
        "        [\n",
        "            \"awards/items/quantity\",\n",
        "            \"awards/items/unit/value/amount\",\n",
        "        ],\n",
        "        [\n",
        "            \"contracts/items/quantity\",\n",
        "            \"contracts/items/unit/value/amount\",\n",
        "        ],\n",
        "    ],\n",
        "    \"when\": [\n",
        "        \"tender/tenderPeriod/endDate\",\n",
        "        \"awards/date\",\n",
        "        \"contracts/dateSigned\",\n",
        "    ],\n",
        "    \"how\": [\n",
        "        \"tender/procurementMethod\",\n",
        "        \"tender/procurementMethodDetails\",\n",
        "    ],\n",
        "}\n",
        "\n",
        "\n",
        "def get_indicators_dictionary(fields_list):\n",
        "    \"\"\"\n",
        "    Check which alternative fields are available for indicators.\n",
        "\n",
        "    For example, the number of tenderers can use either `tender/numberOfTenderers` or `tender/tenderers/id`.\n",
        "    \"\"\"\n",
        "    # U002\n",
        "    buyer = [\"buyer/name\", \"buyer/id\"]\n",
        "    procuring = [\"tender/procuringEntity/name\", \"tender/procuringEntity/id\"]\n",
        "    parties = [\"parties/identifier/name\", \"parties/identifier/id\", \"parties/roles\"]\n",
        "    if not any(item not in fields_list for item in buyer):\n",
        "        buyer_var = buyer\n",
        "    elif not any(item not in fields_list for item in procuring):\n",
        "        buyer_var = procuring\n",
        "    elif not any(item not in fields_list for item in parties):\n",
        "        buyer_var = parties\n",
        "    else:\n",
        "        buyer_var = buyer\n",
        "\n",
        "    # U003\n",
        "    if \"tender/tenderers/id\" in fields_list:\n",
        "        bidders_val = \"tender/tenderers/id\"\n",
        "    elif \"bids/details/tenderers/id\" in fields_list:\n",
        "        bidders_val = \"bids/details/tenderers/id\"\n",
        "    else:\n",
        "        bidders_val = \"tender/tenderers/id\"\n",
        "\n",
        "    # U008\n",
        "    if \"tender/items/classification/id\" in fields_list and \"tender/items/classification/scheme\" in fields_list:\n",
        "        items_val = [\"tender/items/classification/id\", \"tender/items/classification/scheme\"]\n",
        "    elif \"awards/items/classification/id\" in fields_list and \"awards/items/classification/scheme\" in fields_list:\n",
        "        items_val = [\"awards/items/classification/id\", \"awards/items/classification/scheme\"]\n",
        "    elif \"contracts/items/classification/id\" in fields_list and \"contractsitems/classification/scheme\" in fields_list:\n",
        "        items_val = [\"contracts/items/classification/id\", \"contracts/items/classification/scheme\"]\n",
        "    else:\n",
        "        items_val = [\"tender/items/classification/id\", \"tender/items/classification/scheme\"]\n",
        "\n",
        "    # U012\n",
        "    if \"contracts/id\" in fields_list and \"contracts/status\" in fields_list:\n",
        "        awards_val = [\"contracts/id\", \"contracts/status\"]\n",
        "    elif \"awards/id\" in fields_list and \"awards/status\" in fields_list:\n",
        "        awards_val = [\"awards/id\", \"awards/status\"]\n",
        "    else:\n",
        "        awards_val = [\"contracts/id\", \"contracts/status\"]\n",
        "\n",
        "    # U013, UC14\n",
        "    awards = [\"awards/id\", \"awards/status\", \"awards/value/amount\", \"awards/value/currency\"]\n",
        "    contracts = [\"contracts/id\", \"contracts/status\", \"contracts/value/amount\", \"contracts/value/currency\"]\n",
        "    if not any(item not in fields_list for item in contracts):\n",
        "        awards_val2 = contracts\n",
        "    elif not any(item not in fields_list for item in awards):\n",
        "        awards_val2 = awards\n",
        "    else:\n",
        "        awards_val2 = contracts\n",
        "\n",
        "    # U015\n",
        "    if \"tender/numberOfTenderers\" in fields_list:\n",
        "        bidders_val2 = \"tender/numberOfTenderers\"\n",
        "    elif \"tender/tenderers/id\" in fields_list:\n",
        "        bidders_val2 = \"tender/tenderers/id\"\n",
        "    elif \"bids/details/tenderers/id\" in fields_list:\n",
        "        bidders_val2 = \"bids/details/tenderers/id\"\n",
        "    else:\n",
        "        bidders_val2 = \"tender/numberOfTenderers\"\n",
        "\n",
        "    # U034\n",
        "    aw = [\n",
        "        \"awards/status\",\n",
        "        \"awards/value/amount\",\n",
        "        \"awards/value/currency\",\n",
        "        \"awards/items/classification/id\",\n",
        "        \"awards/items/classification/scheme\",\n",
        "    ]\n",
        "    con = [\n",
        "        \"contracts/status\",\n",
        "        \"contracts/value/amount\",\n",
        "        \"contracts/value/currency\",\n",
        "        \"contracts/items/classification/id\",\n",
        "        \"contracts/items/classification/scheme\",\n",
        "    ]\n",
        "    if not any(item not in fields_list for item in aw):\n",
        "        awards_val3 = aw\n",
        "    elif not any(item not in fields_list for item in con):\n",
        "        awards_val3 = con\n",
        "    else:\n",
        "        awards_val3 = aw\n",
        "\n",
        "    # U042\n",
        "    if \"awards/status\" in fields_list:\n",
        "        awards_val4 = \"awards/status\"\n",
        "    elif \"contracts/status\" in fields_list:\n",
        "        awards_val4 = \"contracts/status\"\n",
        "    else:\n",
        "        awards_val4 = \"awards/status\"\n",
        "\n",
        "    # U061\n",
        "    if \"contracts/period/startDate\" in fields_list:\n",
        "        contract_date = \"contracts/period/startDate\"\n",
        "    elif \"awards/contractPeriod/startDate\" in fields_list:\n",
        "        contract_date = \"awards/contractPeriod/startDate\"\n",
        "    else:\n",
        "        contract_date = \"contracts/period/startDate\"\n",
        "\n",
        "    # U065\n",
        "    aw2 = [\"awards/items/quantity\", \"awards/items/unit\"]\n",
        "    con2 = [\"contracts/items/quantity\", \"contracts/items/unit\"]\n",
        "    if not any(item not in fields_list for item in aw):\n",
        "        awards_val5 = aw2\n",
        "    elif not any(item not in fields_list for item in con):\n",
        "        awards_val5 = con2\n",
        "    else:\n",
        "        awards_val5 = aw2\n",
        "\n",
        "    # U066, U067\n",
        "    if \"planning/budget/amount/amount\" in fields_list and \"planning/budget/amount/currency\":\n",
        "        planning = [\"planning/budget/amount/amount\", \"planning/budget/amount/currency\"]\n",
        "    elif \"tender/value/amount\" in fields_list and \"tender/value/currency\":\n",
        "        planning = [\"tender/value/amount\", \"tender/value/currency\"]\n",
        "    else:\n",
        "        planning = [\"planning/budget/amount/amount\", \"planning/budget/amount/currency\"]\n",
        "\n",
        "    return {\n",
        "        \"Total number of procedures\": [[\"U001\"], [\"ocid\"]],\n",
        "        \"Total number of procuring entities\": [[\"U002\"], [\"ocid\", *buyer_var]],\n",
        "        \"Total number of unique bidders\": [[\"U003\"], [\"ocid\", bidders_val]],\n",
        "        \"Total number of awarded suppliers\": [\n",
        "            [\"U004\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", \"awards/status\"],\n",
        "        ],\n",
        "        \"Total number of procedures by year or month\": [[\"U005\"], [\"ocid\", \"date\"]],\n",
        "        \"Total value awarded\": [[\"U006\"], [\"ocid\", \"awards/status\", \"awards/value/amount\", \"awards/value/currency\"]],\n",
        "        \"Share of procedures by status\": [[\"U007\"], [\"ocid\", \"tender/status\"]],\n",
        "        \"Number of procedures by item type\": [[\"U008\"], [\"ocid\", *items_val]],\n",
        "        \"Proportion of procedures by procurement category\": [[\"U009\"], [\"ocid\", \"tender/mainProcurementCategory\"]],\n",
        "        \"Percent of tenders by procedure type\": [[\"U010\"], [\"ocid\", \"tender/procurementMethod\"]],\n",
        "        \"Percent of tenders awarded by means of competitive procedures\": [\n",
        "            [\"U011\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", \"awards/status\"],\n",
        "        ],\n",
        "        \"Percent of contracts awarded under each procedure type\": [\n",
        "            [\"U012\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", *awards_val],\n",
        "        ],\n",
        "        \"Total contracted value awarded under each procedure type\": [\n",
        "            [\"U013\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", *awards_val2],\n",
        "        ],\n",
        "        \"Total awarded value of tenders awarded by means of competitive procedures\": [\n",
        "            [\"U014\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", *awards_val2],\n",
        "        ],\n",
        "        \"Proportion of single bid tenders\": [[\"U015\"], [\"ocid\", \"tender/procurementMethod\", bidders_val2]],\n",
        "        \"Proportion of value awarded in single bid tenders vs competitive tenders\": [\n",
        "            [\"U016\"],\n",
        "            [\n",
        "                \"ocid\",\n",
        "                \"tender/procurementMethod\",\n",
        "                \"awards/status\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "                bidders_val2,\n",
        "            ],\n",
        "        ],\n",
        "        \"Mean number of bidders per tender\": [[\"U017\"], [\"ocid\", \"tender/procurementMethod\", bidders_val2]],\n",
        "        \"Median number of bidders per tender\": [[\"U018\"], [\"ocid\", \"tender/procurementMethod\", bidders_val2]],\n",
        "        \"Mean number of bidders by item type\": [\n",
        "            [\"U019\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", bidders_val2, *items_val],\n",
        "        ],\n",
        "        \"Number of suppliers by item type\": [\n",
        "            [\"U020\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", *items_val],\n",
        "        ],\n",
        "        \"Number of new bidders in a system \": [\n",
        "            [\"U021\"],\n",
        "            [\"tender/id\", \"tender/tenderers/id\", \"tender/tenderPeriod/startDate\"],\n",
        "        ],\n",
        "        \"Percent of new bidders to all bidders \": [\n",
        "            [\"U022\"],\n",
        "            [\"tender/id\", \"tender/tenderers/id\", \"tender/tenderPeriod/startDate\"],\n",
        "        ],\n",
        "        \"Percent of tenders with at least three participants deemed qualified\": [\n",
        "            [\"U023\"],\n",
        "            [\"ocid\", \"bids/details/tenderers/id\", \"bids/details/id\", \"bids/details/status\"],\n",
        "        ],\n",
        "        \"Mean percent of bids which are disqualified\": [\n",
        "            [\"U024\"],\n",
        "            [\"tender/id\", \"bids/details/id\", \"bids/details/status\"],\n",
        "        ],\n",
        "        \"Percent of contracts awarded to top 10 suppliers with largest contracted totals\": [\n",
        "            [\"U025\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", *awards_val2],\n",
        "        ],\n",
        "        \"Mean number of unique suppliers per buyer\": [\n",
        "            [\"U026\"],\n",
        "            [\"ocid\", \"awards/suppliers/id\", \"awards/suppliers/name\", *buyer_var],\n",
        "        ],\n",
        "        \"Number of new awarded suppliers \": [\n",
        "            [\"U027\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", \"awards/date\"],\n",
        "        ],\n",
        "        \"Percent of awards awarded to new suppliers\": [\n",
        "            [\"U028\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", \"awards/date\"],\n",
        "        ],\n",
        "        \"Total awarded value awarded to new suppliers\": [\n",
        "            [\"U029\"],\n",
        "            [\n",
        "                \"awards/id\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/date\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of new suppliers to all suppliers\": [\n",
        "            [\"U030\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", \"awards/date\"],\n",
        "        ],\n",
        "        \"Percent of growth of new awarded suppliers in a system\": [\n",
        "            [\"U031\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", \"awards/date\"],\n",
        "        ],\n",
        "        \"Percent of total awarded value awarded to recurring suppliers\": [\n",
        "            [\"U032\"],\n",
        "            [\n",
        "                \"awards/id\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/date\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Mean number of bids necessary to win\": [\n",
        "            [\"U033\"],\n",
        "            [\"ocid\", \"tender/tenderers/id\", \"awards/suppliers/id\", \"awards/suppliers/name\"],\n",
        "        ],\n",
        "        \"Market concentration, market share of the largest company in the market\": [\n",
        "            [\"U034\"],\n",
        "            [\"awards/suppliers/id\", \"awards/suppliers/name\", *awards_val3],\n",
        "        ],\n",
        "        \"Proportion of contracts awarded by supplier by non competitive procedures\": [\n",
        "            [\"U035\"],\n",
        "            [\"ocid\", \"tender/procurementMethod\", \"awards/status\", \"awards/suppliers/id\", \"awards/suppliers/name\"],\n",
        "        ],\n",
        "        \"Region of the supplier\": [[\"U036\"], [\"parties/roles\", \"parties/identifier/id\", \"parties/address/region\"]],\n",
        "        \"Number of bids submitted by supplier\": [[\"U037\"], [\"awards/suppliers/id\", bidders_val]],\n",
        "        \"Success rate of bidders\": [\n",
        "            [\"U038\"],\n",
        "            [\"ocid\", \"tender/tenderers/id\", \"awards/suppliers/id\", \"awards/suppliers/name\"],\n",
        "        ],\n",
        "        \"Number of unique items classifications awarded by supplier\": [\n",
        "            [\"U039\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", *items_val],\n",
        "        ],\n",
        "        \"Total value awarded by supplier\": [[\"U040\"], [\"awards/suppliers/id\", \"awards/suppliers/name\", *awards_val2]],\n",
        "        \"Share of total value awarded by supplier\": [\n",
        "            [\"U041\"],\n",
        "            [\"awards/suppliers/id\", \"awards/suppliers/name\", *awards_val2],\n",
        "        ],\n",
        "        \"Total number of contracts awarded by supplier\": [\n",
        "            [\"U042\"],\n",
        "            [\"awards/id\", \"awards/suppliers/id\", \"awards/suppliers/name\", awards_val4],\n",
        "        ],\n",
        "        \"Number of procuring entities by supplier\": [\n",
        "            [\"U043\"],\n",
        "            [\"ocid\", \"awards/suppliers/id\", \"awards/suppliers/name\", *buyer_var],\n",
        "        ],\n",
        "        \"Share of single bid awards by supplier\": [\n",
        "            [\"U044\"],\n",
        "            [\n",
        "                \"ocid\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/status\",\n",
        "                \"tender/procurementMethod\",\n",
        "                bidders_val2,\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of tenders with linked procurement plans\": [[\"U045\"], [\"tender/id\", \"tender/documents/documentType\"]],\n",
        "        \"Percent of contracts which publish information on debarments\": [\n",
        "            [\"U046\"],\n",
        "            [\"contracts/id\", \"contracts/implementation/documents/documentType\"],\n",
        "        ],\n",
        "        \"The percent of tenders for which the tender documentation was added after publication of the announcement \": [\n",
        "            [\"U047\"],\n",
        "            [\n",
        "                \"tender/id\",\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/documents/datePublished\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Mean number of contract amendments per buyer\": [\n",
        "            [\"U048\"],\n",
        "            [\"ocid\", \"contracts/id\", \"contracts/amendments\", *buyer_var],\n",
        "        ],\n",
        "        \"Percent of tenders which have been closed for more than 30 days, but whose basic awards information is not published\": [\n",
        "            [\"U049\"],\n",
        "            [\n",
        "                \"tender/id\",\n",
        "                \"tender/tenderPeriod/endDate\",\n",
        "                \"awards/id\",\n",
        "                \"awards/date\",\n",
        "                \"awards/status\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of awards which are older than 30 days, but whose contract is not published\": [\n",
        "            [\"U050\"],\n",
        "            [\n",
        "                \"awards/id\",\n",
        "                \"awards/date\",\n",
        "                \"contracts/awardID\",\n",
        "                \"contracts/status\",\n",
        "                \"contracts/dateSigned\",\n",
        "                \"contracts/documents/documentType\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of tenders that do not specify place of delivery\": [\n",
        "            [\"U051\"],\n",
        "            [\"ocid\", \"tender/items/deliveryLocation\", \"tender/items/deliveryAddress\"],\n",
        "        ],\n",
        "        \"Percent of tenders that do not specify date of delivery\": [\n",
        "            [\"U052\"],\n",
        "            [\n",
        "                \"tender/milestones/id\",\n",
        "                \"tender/milestones/type\",\n",
        "                \"tender/milestones/description\",\n",
        "                \"tender/milestones/dueDate\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of tenders with short titles for example fewer than 10 characters in the title\": [\n",
        "            [\"U053\"],\n",
        "            [\"tender/id\", \"tender/title\"],\n",
        "        ],\n",
        "        \"Percent of tenders with short descriptions for instance fewer than 30 characters in the description\": [\n",
        "            [\"U054\"],\n",
        "            [\"tender/id\", \"tender/description\"],\n",
        "        ],\n",
        "        \"Percent of tenders that do not include detailed item codes or item descriptions\": [\n",
        "            [\"U055\"],\n",
        "            [\"tender/id\", \"tender/items/classification/id\", \"tender/items/classification/scheme\"],\n",
        "        ],\n",
        "        \"Percent of contracts that do not have amendments\": [[\"U056\"], [\"contracts/id\", \"contracts/amendments\"]],\n",
        "        \"Percent of contracts which publish contract implementation details financial\": [\n",
        "            [\"U057\"],\n",
        "            [\n",
        "                \"contracts/implementation/transactions/id\",\n",
        "                \"contracts/implementation/transactions/value/amount\",\n",
        "                \"contracts/implementation/transactions/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of contracts which publish contract implementation details physical\": [\n",
        "            [\"U058\"],\n",
        "            [\n",
        "                \"contracts/implementation/milestones/type\",\n",
        "                \"contracts/implementation/milestones/id\",\n",
        "                \"contracts/implementation/milestones/dueDate\",\n",
        "                \"contracts/implementation/milestones/status\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Average duration of tendering period days\": [\n",
        "            [\"U059\"],\n",
        "            [\"ocid\", \"tender/tenderPeriod/startDate\", \"tender/tenderPeriod/endDate\"],\n",
        "        ],\n",
        "        \"Average duration of decision period days\": [[\"U060\"], [\"ocid\", \"tender/tenderPeriod/endDate\", \"awards/date\"]],\n",
        "        \"Average days from award date to start of implementation\": [\n",
        "            [\"U061\"],\n",
        "            [\"awards/id\", \"awards/date\", contract_date],\n",
        "        ],\n",
        "        \"Days between award date and tender start date\": [\n",
        "            [\"U062\"],\n",
        "            [\"ocid\", \"tender/tenderPeriod/startDate\", \"awards/date\"],\n",
        "        ],\n",
        "        \"Percent of canceled tenders to awarded tenders\": [[\"U063\"], [\"ocid\", \"tender/status\", \"awards/status\"]],\n",
        "        \"Percent of contracts which are canceled\": [[\"U064\"], [\"contracts/id\", \"contracts/status\"]],\n",
        "        \"Price variation of same item across all awards\": [[\"U065\"], awards_val3 + awards_val5],\n",
        "        \"Percent of contracts that exceed budget\": [\n",
        "            [\"U066\"],\n",
        "            [\"ocid\", \"contracts/status\", \"contracts/value/amount\", \"contracts/value/currency\", *planning],\n",
        "        ],\n",
        "        \"Mean percent overrun of contracts that exceed budget\": [\n",
        "            [\"U067\"],\n",
        "            [\"ocid\", \"contracts/status\", \"contracts/value/amount\", \"contracts/value/currency\", *planning],\n",
        "        ],\n",
        "        \"Total percent savings difference between budget and contract value\": [\n",
        "            [\"U068\"],\n",
        "            [\n",
        "                \"ocid\",\n",
        "                \"planning/budget/amount/amount\",\n",
        "                \"planning/budget/amount/currency\",\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Total percent savings difference between tender value estimate and contract value\": [\n",
        "            [\"U069\"],\n",
        "            [\n",
        "                \"ocid\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Percent of contracts completed on time \": [\n",
        "            [\"U070\"],\n",
        "            [\"contracts/id\", \"contracts/period/endDate\", \"contracts/status\"],\n",
        "        ],\n",
        "        \"Share of contracts whose milestones are completed on time\": [\n",
        "            [\"U071\"],\n",
        "            [\n",
        "                \"contracts/id\",\n",
        "                \"contracts/implementation/milestones/dueDate\",\n",
        "                \"contracts/implementation/milestones/dateMet\",\n",
        "            ],\n",
        "        ],\n",
        "    }\n",
        "\n",
        "\n",
        "def usability_checks(fields_list, indicators_dic):\n",
        "    \"\"\"\n",
        "    Return a table of the usability checks.\n",
        "\n",
        "    It indicates if the fields needed to calculate a particular indicator are present.\n",
        "    Set check_coverage=True to check for coverage.\n",
        "    \"\"\"\n",
        "    results_list = []\n",
        "    missing_fields = []\n",
        "\n",
        "    for i in indicators_dic.values():\n",
        "        check = any(item not in fields_list for item in i[1])\n",
        "        result = \"missing fields\" if check else \"possible to calculate\"\n",
        "        missing = [i[1][j] for j in range(len(i[1])) if i[1][j] not in fields_list]\n",
        "        missing_fields.append(missing)\n",
        "        results_list.append(result)\n",
        "\n",
        "    # Generate dataframe\n",
        "\n",
        "    indicatordf = pd.DataFrame(\n",
        "        list(\n",
        "            zip(\n",
        "                list(indicators_dic),\n",
        "                [indicators_dic[i][0] for i in indicators_dic],\n",
        "                [indicators_dic[i][1:] for i in indicators_dic],\n",
        "                strict=True,\n",
        "            )\n",
        "        ),\n",
        "        columns=[\"indicator\", \"U_id\", \"fields needed\"],\n",
        "    )\n",
        "    indicatordf[\"U_id\"] = indicatordf[\"U_id\"].apply(lambda x: \", \".join(map(str, x)))\n",
        "    indicatordf[\"fields needed\"] = indicatordf[\"fields needed\"].astype(str).str.replace(r\"\\[|\\]|'\", \"\", regex=True)\n",
        "    indicatordf[\"calculation\"] = results_list\n",
        "    indicatordf[\"missing fields\"] = missing_fields\n",
        "    indicatordf[\"missing fields\"] = indicatordf[\"missing fields\"].apply(lambda x: \", \".join(map(str, x)))\n",
        "\n",
        "    return indicatordf\n",
        "\n",
        "\n",
        "def check_usability_indicators(lang, result):\n",
        "\n",
        "    gc = authenticate_gspread()\n",
        "\n",
        "    if lang.value == \"English\":\n",
        "        language = \"Use case guide: Indicators linked to OCDS #public\"\n",
        "    else:\n",
        "        language = \"[ES] of Use case guide: Indicators linked to OCDS #public\"\n",
        "\n",
        "    worksheet = gc.open(language).sheet1\n",
        "\n",
        "    # get_all_values gives a list of rows.\n",
        "    rows = worksheet.get_all_values()\n",
        "    # Convert to a DataFrame and render.\n",
        "\n",
        "    indicators = pd.DataFrame(rows)\n",
        "    indicators = indicators.rename(columns=indicators.iloc[0]).drop(indicators.index[0])\n",
        "\n",
        "    if lang.value == \"English\":\n",
        "        indicatorsdf = indicators.iloc[:, [0, 3, 4, 9]]\n",
        "        result_final = result.merge(indicatorsdf, on=\"U_id\")\n",
        "    else:\n",
        "        indicatorsdf = indicators.iloc[:, [0, 3, 4, 5, 9]]\n",
        "        result_final = indicatorsdf.merge(result, on=\"U_id\").drop([\"indicator\"], axis=1)\n",
        "        result_final = result_final.rename(\n",
        "            columns={\n",
        "                \"fields needed\": \"Campos necesarios\",\n",
        "                \"calculation\": \"¿Se puede calcular?\",\n",
        "                \"missing fields\": \"Campos faltantes\",\n",
        "                \"coverage\": \"Cobertura\",\n",
        "            },\n",
        "        )\n",
        "        result_final = result_final.replace(\n",
        "            {\"¿Se puede calcular?\": {\"possible to calculate\": \"sí\", \"missing fields\": \"campos faltantes\"}}\n",
        "        )\n",
        "    return result_final\n",
        "\n",
        "\n",
        "def check_red_flags_indicators(result):\n",
        "\n",
        "    gc = authenticate_gspread()\n",
        "\n",
        "    worksheet = gc.open(\"Red Flags to OCDS Mapping #public\").get_worksheet(1)\n",
        "\n",
        "    # get_all_values gives a list of rows.\n",
        "    rows = worksheet.get_all_values()\n",
        "    # Convert to a DataFrame and render.\n",
        "\n",
        "    indicators = pd.DataFrame(rows)\n",
        "    indicators = indicators.rename(columns=indicators.iloc[0]).drop(indicators.index[0])\n",
        "    indicatorsdf = indicators.iloc[:, [0, 5, 6, 7]]\n",
        "\n",
        "    return result.merge(indicatorsdf, on=\"R_id\")\n",
        "\n",
        "\n",
        "def is_relevant(field_list):\n",
        "    final_result = []\n",
        "    for key, value in RELEVANT_RULES.items():\n",
        "        rule_result = {\"rule\": key, \"possible_to_calculate\": \"No\", \"available_fields\": [], \"missing_fields\": []}\n",
        "        for field in value:\n",
        "            if isinstance(field, str):\n",
        "                if field in field_list:\n",
        "                    rule_result[\"possible_to_calculate\"] = \"Yes\"\n",
        "                    rule_result[\"available_fields\"].append(field)\n",
        "                else:\n",
        "                    rule_result[\"missing_fields\"].append(field)\n",
        "            else:\n",
        "                missing = list(filter(lambda item: item not in field_list, field))\n",
        "                rule_result[\"available_fields\"].extend(list(filter(lambda item: item in field_list, field)))\n",
        "                rule_result[\"missing_fields\"].extend(missing)\n",
        "                if not missing:\n",
        "                    rule_result[\"possible_to_calculate\"] = \"Yes\"\n",
        "        final_result.append(rule_result)\n",
        "\n",
        "    final_result = pd.DataFrame(final_result)\n",
        "    relevant = (final_result[\"possible_to_calculate\"] == \"Yes\").all()\n",
        "    return relevant, final_result\n",
        "\n",
        "\n",
        "def get_coverage(indicators_dic):\n",
        "    coverage = []\n",
        "    for i in indicators_dic.values():\n",
        "        fields = [item for sublist in i for item in sublist][1:]\n",
        "        result = calculate_coverage(fields, \"release_summary\")\n",
        "        result_value = pd.to_numeric(result[\"total_percentage\"][0])\n",
        "        coverage.append(result_value)\n",
        "    return coverage\n",
        "\n",
        "\n",
        "def most_common_fields_to_calculate_indicators(indicators_dict, fields_table):\n",
        "    fields = list(indicators_dict.values())\n",
        "    fields = [item[1:] for item in fields]\n",
        "    flat_list = [item for sublist in [item for sublist in fields for item in sublist] for item in sublist]\n",
        "    fields_list = Counter(flat_list)\n",
        "\n",
        "    fields_count = (\n",
        "        pd.DataFrame.from_dict(fields_list, orient=\"index\")\n",
        "        .reset_index()\n",
        "        .rename(columns={\"index\": \"field\", 0: \"number of indicators\"})\n",
        "    )\n",
        "\n",
        "    fields_count = fields_count.sort_values(\"number of indicators\", ascending=False).reset_index(drop=True)\n",
        "    fields_count[\"published\"] = np.where(fields_count[\"field\"].isin(fields_table[\"path\"]), \"yes\", \"no\")\n",
        "\n",
        "    return fields_count\n",
        "\n",
        "\n",
        "def get_usability_language_select_box():\n",
        "    style = {\"description_width\": \"initial\"}\n",
        "    languages = [\"Spanish\", \"English\"]\n",
        "    return widgets.Dropdown(options=languages, description=\"language\", style=style)\n",
        "\n",
        "\n",
        "def get_red_flags_dictionary(fields_list):\n",
        "    # buyers\n",
        "    buyer = [\"buyer/name\", \"buyer/id\"]\n",
        "    procuring = [\"tender/procuringEntity/name\", \"tender/procuringEntity/id\"]\n",
        "    parties = [\"parties/identifier/name\", \"parties/identifier/id\", \"parties/roles\"]\n",
        "    if not any(item not in fields_list for item in buyer):\n",
        "        buyer_var = buyer\n",
        "    elif not any(item not in fields_list for item in procuring):\n",
        "        buyer_var = procuring\n",
        "    elif not any(item not in fields_list for item in parties):\n",
        "        buyer_var = parties\n",
        "    else:\n",
        "        buyer_var = buyer\n",
        "\n",
        "    # number of tendereres\n",
        "    if \"tender/numberOfTenderers\" in fields_list:\n",
        "        bidders_val2 = \"tender/numberOfTenderers\"\n",
        "    elif \"tender/tenderers/id\" in fields_list:\n",
        "        bidders_val2 = \"tender/tenderers/id\"\n",
        "    elif \"bids/details/tenderers/id\" in fields_list:\n",
        "        bidders_val2 = \"bids/details/tenderers/id\"\n",
        "    else:\n",
        "        bidders_val2 = \"tender/numberOfTenderers\"\n",
        "\n",
        "    # bidders\n",
        "    if \"tender/tenderers/id\" in fields_list:\n",
        "        bidders_val = \"tender/tenderers/id\"\n",
        "    elif \"bids/details/tenderers/id\" in fields_list:\n",
        "        bidders_val = \"bids/details/tenderers/id\"\n",
        "    else:\n",
        "        bidders_val = \"tender/tenderers/id\"\n",
        "\n",
        "    # suppliers\n",
        "    supplier = [\"awards/suppliers/id\"]\n",
        "    parties = [\"parties/identifier/id\", \"parties/roles\"]\n",
        "    if not any(item not in fields_list for item in supplier):\n",
        "        sup_var = buyer\n",
        "    elif not any(item not in fields_list for item in parties):\n",
        "        sup_var = parties\n",
        "    else:\n",
        "        sup_var = supplier\n",
        "\n",
        "    if \"awards/suppliers/id\" in fields_list:\n",
        "        sup_var = \"awards/suppliers/id\"\n",
        "    elif \"bids/details/tenderers/id\" in fields_list:\n",
        "        bidders_val = \"bids/details/tenderers/id\"\n",
        "    else:\n",
        "        bidders_val = \"tender/tenderers/id\"\n",
        "\n",
        "    # awards and contracts fields\n",
        "    aw = [\n",
        "        \"awards/status\",\n",
        "        \"awards/date\",\n",
        "        \"awards/value/amount\",\n",
        "        \"awards/value/currency\",\n",
        "        \"awards/items/classification/id\",\n",
        "        \"awards/items/classification/scheme\",\n",
        "    ]\n",
        "    con = [\n",
        "        \"contracts/status\",\n",
        "        \"contracts/dateSigned\",\n",
        "        \"contracts/value/amount\",\n",
        "        \"contracts/value/currency\",\n",
        "        \"contracts/items/classification/id\",\n",
        "        \"contracts/items/classification/scheme\",\n",
        "    ]\n",
        "    if not any(item not in fields_list for item in aw):\n",
        "        awards_val3 = aw\n",
        "    elif not any(item not in fields_list for item in con):\n",
        "        awards_val3 = con\n",
        "    else:\n",
        "        awards_val3 = aw\n",
        "\n",
        "    return {\n",
        "        \"Planning documents unavailable\": [[\"R001\"], [\"planning/documents/documentType\"]],\n",
        "        \"Manipulation of procurement thresholds\": [[\"R002\"], [\"planning/budget/amount\", \"tender/procurementMethod\"]],\n",
        "        \"Short submission period\": [\n",
        "            [\"R003\"],\n",
        "            [\"tender/tenderPeriod/startDate\", \"tender/tenderPeriod/endDate\", \"tender/procurementMethod\"],\n",
        "        ],\n",
        "        \"Failure to adequately advertise the request for bids or proposals\": [\n",
        "            [\"R004\"],\n",
        "            [\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/documents/datePublished\",\n",
        "                \"tender/tenderPeriod/startDate\",\n",
        "                \"tender/tenderPeriod/endDate\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Key tender information and documents are not available\": [\n",
        "            [\"R005\"],\n",
        "            [\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/documents/datePublished\",\n",
        "                \"tender/tenderPeriod/startDate\",\n",
        "                \"tender/tenderPeriod/endDate\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Unreasonable prequalification requirements\": [[\"R006\"], [\"tender/documents/documentType\"]],\n",
        "        \"Vague, ambiguous, unreasonably strict or narrow, or incomplete specifications\": [\n",
        "            [\"R007\"],\n",
        "            [\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/items/classification/id\",\n",
        "                *buyer_var,\n",
        "                \"tender/value/amount\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Failure to make bidding documents available to all bidders\": [\n",
        "            [\"R008\"],\n",
        "            [\n",
        "                \"tender/documents/documentType\",\n",
        "                \"tender/documents/datePublished\",\n",
        "                \"tender/tenderPeriod/startDate\",\n",
        "                \"tender/tenderPeriod/endDate\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Buyer increases the cost of the bidding documents\": [\n",
        "            [\"R009\"],\n",
        "            [\"tender/participationFees/value/amount\", \"tender/participationFees/value/currency\", \"date\"],\n",
        "        ],\n",
        "        \"Bundling tenders in unreasonably large or small amounts to discourage or eliminate certain bidders\": [\n",
        "            [\"R010\"],\n",
        "            [\"tender/value/amount\", \"tender/value/currency\", \"tender/procurementMethod\", *buyer_var],\n",
        "        ],\n",
        "        \"Splitting purchases to avoid procurement thresholds\": [\n",
        "            [\"R011\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"tender/tenderPeriod/startDate\",\n",
        "                *buyer_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"Direct awards in contravention to the provisions of the procurement plan\": [\n",
        "            [\"R012\"],\n",
        "            [\"tender/procurementMethod\", \"tender/procurementMethodDetails\", \"planning/documents/documentType\"],\n",
        "        ],\n",
        "        \"Tender is invitation only\": [\n",
        "            [\"R013\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/procurementMethodDetails\",\n",
        "                \"tender/procurementMethodRationale\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/description\",\n",
        "                \"tender/title\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Short time between tender advertising and bid opening\": [\n",
        "            [\"R014\"],\n",
        "            [\"tender/tenderPeriod/startDate\", \"tender/bidOpening/date\", \"tender/procurementMethod\"],\n",
        "        ],\n",
        "        \"Long time between bid opening and bid evaluation\": [\n",
        "            [\"R015\"],\n",
        "            [\"tender/bidOpening/date\", \"tender/awardPeriod/startDate\", \"tender/procurementMethod\"],\n",
        "        ],\n",
        "        \"Tender value is higher or lower than average for this item category\": [\n",
        "            [\"R016\"],\n",
        "            [\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                \"tender/items/quantity\",\n",
        "                \"tender/items/unit\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Unreasonably low or high line item\": [\n",
        "            [\"R017\"],\n",
        "            [\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"tender/items/unit\",\n",
        "                \"tender/items/quantity\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Single bid received\": [[\"R018\"], [\"tender/procurementMethod\", bidders_val2]],\n",
        "        \"Low number of bidders for item and procuring entity\": [\n",
        "            [\"R019\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                *buyer_var,\n",
        "                bidders_val2,\n",
        "            ],\n",
        "        ],\n",
        "        \"Tender has a complaint\": [[\"R020\"], [\"complaints/date\", \"complaints/description\", \"complaints/documents\"]],\n",
        "        \"Inappropriate evaluation criteria or procedures\": [[\"R021\"], [\"tender/documents/documentType\"]],\n",
        "        \"Wide disparity in bid prices\": [\n",
        "            [\"R022\"],\n",
        "            [\"bids/details/id\", \"bids/details/value/amount\", \"bids/details/value/currency\"],\n",
        "        ],\n",
        "        \"Fixed multiple bid prices\": [\n",
        "            [\"R023\"],\n",
        "            [\"bids/details/id\", \"bids/details/value/amount\", \"bids/details/value/currency\"],\n",
        "        ],\n",
        "        \"Price close to winning bid\": [\n",
        "            [\"R024\"],\n",
        "            [\"bids/details/id\", \"bids/details/value/amount\", \"bids/details/value/currency\"],\n",
        "        ],\n",
        "        \"Excessive unsuccessful bids\": [[\"R025\"], [\"awards/suppliers/id\", bidders_val]],\n",
        "        \"Prevalence of joint bid patterns (consortia)\": [[\"R026\"], [\"ocid\", bidders_val]],\n",
        "        \"Potential bidders make agreements not to bid because of Collusion arrangements (Missing bidders)\": [\n",
        "            [\"R027\"],\n",
        "            [\"tender/items/classification/id\", \"tender/items/classification/scheme\", bidders_val],\n",
        "        ],\n",
        "        \"Identical bid prices\": [\n",
        "            [\"R028\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Losing bids are round numbers\": [\n",
        "            [\"R029\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "                \"bids/details/status\",\n",
        "                \"bids/awards/relatedBid\",\n",
        "                *sup_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"Late bid won\": [\n",
        "            [\"R030\"],\n",
        "            [\"bids/details/id\", \"bids/details/date\", \"bids/details/status\", \"tender/tenderPeriod/endDate\"],\n",
        "        ],\n",
        "        \"Bid is too close to budget, estimate or preferred solution\": [\n",
        "            [\"R031\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/status\",\n",
        "                \"planning/budget/amount/amount\",\n",
        "                \"tender/value/amount\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Persistently high or increasing bid prices\": [\n",
        "            [\"R032\"],\n",
        "            [\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"tender/items/unit\",\n",
        "                \"tender/items/quantity\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                *buyer_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"Late bidder is the winning bidder\": [\n",
        "            [\"R033\"],\n",
        "            [\"bids/details/id\", \"bids/details/date\", \"bids/details/status\", \"bids/awards/relatedBid\", *sup_var],\n",
        "        ],\n",
        "        \"Bidders submit bids in subsequent re-bidding rounds in same order as in original bid\": [\n",
        "            [\"R034\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/date\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "                \"bids/details/status\",\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/procurementMethod\",\n",
        "                *sup_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"All except winning bid disqualified\": [\n",
        "            [\"R035\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/status\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/awards/relatedBid\",\n",
        "                \"awards/suppliers/id\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Lowest bid disqualified \": [\n",
        "            [\"R036\"],\n",
        "            [\n",
        "                \"tender/awardCriteria\",\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/status\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "                \"bids/details/documents\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Poorly supported disqualifications\": [\n",
        "            [\"R037\"],\n",
        "            [\n",
        "                \"tender/awardCriteria\",\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/status\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "                \"bids/details/documents\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Excessive disqualified bids\": [\n",
        "            [\"R038\"],\n",
        "            [\"bids/details/tenderers/id\", \"bids/details/tenderers/name\", \"bids/details/status\"],\n",
        "            [\n",
        "                \"tender/procuringEntity/name\",\n",
        "                \"buyer/name\",\n",
        "                \"parties/identifier/id\",\n",
        "                \"parties/identifier/name\",\n",
        "                \"parties/roles\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Unanswered bidder questions\": [\n",
        "            [\"R039\"],\n",
        "            [\"tender/enquiries/date\", \"tender/enquiries/dateAnswered\", \"tender/enquiries/answer\"],\n",
        "        ],\n",
        "        \"Close relationships exists between bidder and buyer\": [\n",
        "            [\"R040\"],\n",
        "            [\"parties/id\", \"parties/name\", \"parties/roles\", \"awards/value/amount\"],\n",
        "        ],\n",
        "        \"Physical similarities in documents by different bidders\": [\n",
        "            [\"R041\"],\n",
        "            [\n",
        "                \"bids/documents/title\",\n",
        "                \"bids/documents/description\",\n",
        "                \"bids/documents/documentType\",\n",
        "                \"bids/documents/url\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Supplier (or bidder) has abnormal address or phone number\": [\n",
        "            [\"R042\"],\n",
        "            [\n",
        "                \"parties/roles\",\n",
        "                \"parties/identifier/id\",\n",
        "                \"parties/contactPoint/telephone\",\n",
        "                \"parties/address/streetAddress\",\n",
        "                \"parties/address/postalCode\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Supplier (or bidder)  address is same as project officials\": [\n",
        "            [\"R043\"],\n",
        "            [\n",
        "                \"parties/roles\",\n",
        "                \"parties/identifier/id\",\n",
        "                \"parties/contactPoint/telephone\",\n",
        "                \"parties/address/streetAddress\",\n",
        "                \"parties/address/postalCode\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Business similarities between suppliers (or bidders)\": [\n",
        "            [\"R044\"],\n",
        "            [\n",
        "                \"parties/roles\",\n",
        "                \"parties/identifier/id\",\n",
        "                \"parties/contactPoint/name\",\n",
        "                \"parties/contactPoint/email\",\n",
        "                \"parties/contactPoint/telephone\",\n",
        "                \"parties/address/streetAddress\",\n",
        "                \"parties/address/postalCode\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Supplier does not have internet presence\": [\n",
        "            [\"R047\"],\n",
        "            [\"parties/roles\", \"parties/id\", \"parties/name\", \"parties/contactPoint/url\"],\n",
        "        ],\n",
        "        \"Heterogeneous supplier\": [\n",
        "            [\"R048\"],\n",
        "            [\n",
        "                \"awards/items/classification/id\",\n",
        "                \"awards/items/classification/scheme\",\n",
        "                \"awards/items/quantity\",\n",
        "                \"awards/items/unit\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "            ],\n",
        "        ],\n",
        "        \"High number of direct awards to one bidder\": [\n",
        "            [\"R049\"],\n",
        "            [\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "                \"awards/date\",\n",
        "                \"tender/procurementMethod\",\n",
        "                *buyer_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"One or a few bidders win a disproportionate number of contracts of the same type\": [\n",
        "            [\"R050\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "                \"awards/items/classification/id\",\n",
        "                \"awards/items/classification/scheme\",\n",
        "                \"awards/date\",\n",
        "                \"awards/status\",\n",
        "                *buyer_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"High market concentration\": [\n",
        "            [\"R051\"],\n",
        "            [\"tender/procurementMethod\", \"awards/suppliers/id\", \"awards/suppliers/name\", *awards_val3],\n",
        "        ],\n",
        "        \"Small initial purchase from supplier followed by much larger purchases\": [\n",
        "            [\"R052\"],\n",
        "            [\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/id\",\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/procuringEntity/name\",\n",
        "                *buyer_var,\n",
        "                *awards_val3,\n",
        "            ],\n",
        "        ],\n",
        "        \"The same companies always bid, the same companies always win and the same companies always lose\": [\n",
        "            [\"R053\"],\n",
        "            [\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"tender/items/classification/id\",\n",
        "                \"tender/items/classification/scheme\",\n",
        "                bidders_val,\n",
        "            ],\n",
        "        ],\n",
        "        \"Awards below the competitive bid threshold followed by change orders that exceed the threshold\": [\n",
        "            [\"R054\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "                \"contracts/amendments/description\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Multiple direct awards above or just below the direct award threshold\": [\n",
        "            [\"R055\"],\n",
        "            [\n",
        "                \"tender/procurementMethod\",\n",
        "                \"tender/value/amount\",\n",
        "                \"tender/value/currency\",\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                *buyer_var,\n",
        "            ],\n",
        "        ],\n",
        "        \"The winning bid does not meet the award criteria\": [\n",
        "            [\"R056\"],\n",
        "            [\"tender/awardCriteria\", \"bids/details/status\", \"bids/details/documents\"],\n",
        "        ],\n",
        "        \"Rotation of winning bidders by job, type of work or geographical area\": [\n",
        "            [\"R057\"],\n",
        "            [\n",
        "                \"awards/suppliers/id\",\n",
        "                \"awards/suppliers/name\",\n",
        "                \"awards/value/amount\",\n",
        "                \"awards/value/currency\",\n",
        "                \"awards/items/classification/id\",\n",
        "                \"awards/items/classification/scheme\",\n",
        "                \"parties/address/region\",\n",
        "                \"parties/address/addressDetails/region\",\n",
        "                bidders_val,\n",
        "            ],\n",
        "        ],\n",
        "        \"Heavily discounted bid\": [\n",
        "            [\"R058\"],\n",
        "            [\n",
        "                \"bids/details/id\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/tenderers/id\",\n",
        "                \"bids/details/tenderers/name\",\n",
        "                \"bids/details/status\",\n",
        "                \" bids/awards/relatedBid\",\n",
        "                \"awards/suppliers/id\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Large difference between the award value and final contract amount\": [\n",
        "            [\"R059\"],\n",
        "            [\"awards/id\", \"awards/value/amount\", \"contracts/awardID\", \"contracts/value/amount\"],\n",
        "        ],\n",
        "        \"Large difference between contract price and winning bid price\": [\n",
        "            [\"R060\"],\n",
        "            [\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "                \"bids/details/value/amount\",\n",
        "                \"bids/details/value/currency\",\n",
        "                \"bids/details/id\",\n",
        "                \" bids/details/tenderers/id\",\n",
        "                \"awards/suppliers/id\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Long unexplained delays in contract negotiations or awards\": [\n",
        "            [\"R061\"],\n",
        "            [\"awards/date\", \"contracts/dateSigned\"],\n",
        "        ],\n",
        "        \"Decision period for submitted bids excessively short\": [\n",
        "            [\"R062\"],\n",
        "            [\"tender/tenderPeriod/endDate\", \"awards/date\", \"tender/procurementMethod\"],\n",
        "        ],\n",
        "        \"Decision period for submitted bids excessively long or involved legal challenge\": [\n",
        "            [\"R063\"],\n",
        "            [\"tender/tenderPeriod/endDate\", \"awards/date\", \"tender/procurementMethod\"],\n",
        "        ],\n",
        "        \"Contract is not public\": [[\"R064\"], [\"contracts/documents/documentType\"]],\n",
        "        \"Change orders issued after contract award, reducing or deleting item\": [\n",
        "            [\"R065\"],\n",
        "            [\"contract/amendments/date\", \"contract/amendments/description\", \"contract/amendments/rationale\"],\n",
        "        ],\n",
        "        \"Change orders issued after contract award, extending the line item requirements\": [\n",
        "            [\"R066\"],\n",
        "            [\"contract/amendments/date\", \"contract/amendments/description\", \"contract/amendments/rationale\"],\n",
        "        ],\n",
        "        \"Delivery failure\": [\n",
        "            [\"R067\"],\n",
        "            [\n",
        "                \"contracts/id\",\n",
        "                \"contracts/implementation/milestones/type\",\n",
        "                \"contracts/implementation/milestones/dueDate\",\n",
        "                \"contracts/implementation/milestones/dateMet\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Total payments to a contractor exceed total contract or purchase order amounts\": [\n",
        "            [\"R068\"],\n",
        "            [\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "                \"contracts/implementation/transactions/value/amount\",\n",
        "                \"contracts/implementation/transactions/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Approval of unnecessary change orders to increase the contract price after award\": [\n",
        "            [\"R069\"],\n",
        "            [\n",
        "                \"contracts/amendments/description\",\n",
        "                \"contracts/value/amount\",\n",
        "                \"contracts/value/currency\",\n",
        "                \"contracts/implementation/transactions/value/amount\",\n",
        "                \"contracts/implementation/transactions/value/currency\",\n",
        "            ],\n",
        "        ],\n",
        "        \"Losing bidders are hired as subcontractors or suppliers\": [\n",
        "            [\"R070\"],\n",
        "            [\"awards/hasSubcontracting\", bidders_val],\n",
        "        ],\n",
        "        \"A contractor subcontracts all or most of the work received (indicating it could be a shell company)\": [\n",
        "            [\"R071\"],\n",
        "            [\"awards/hasSubcontracting\", \"awards/subcontracting/minimumPercentage\"],\n",
        "        ],\n",
        "        \"Prevalence of subcontracting\": [[\"R072\"], [\"awards/hasSubcontracting\"]],\n",
        "        \"Discrepancies between work completed and contract specifications\": [\n",
        "            [\"R073\"],\n",
        "            [\n",
        "                \"contracts/status\",\n",
        "                \"tender/documents/documentType\",\n",
        "                \"contracts/documents/documentType\",\n",
        "                \"contracts/implementation/documents/documentType\",\n",
        "            ],\n",
        "        ],\n",
        "    }\n",
        "\n",
        "\n",
        "def redflags_checks(fields_list, indicators_dic, *, check_coverage=False):\n",
        "    results_list = []\n",
        "    missing_fields = []\n",
        "\n",
        "    for i in indicators_dic.values():\n",
        "        check = any(item not in fields_list for item in i[1])\n",
        "        result = \"missing fields\" if check else \"possible to calculate\"\n",
        "        missing = [i[1][j] for j in range(len(i[1])) if i[1][j] not in fields_list]\n",
        "        missing_fields.append(missing)\n",
        "        results_list.append(result)\n",
        "\n",
        "    # Generate dataframe\n",
        "\n",
        "    indicatordf = pd.DataFrame(\n",
        "        list(\n",
        "            zip(\n",
        "                list(indicators_dic),\n",
        "                [indicators_dic[i][0] for i in indicators_dic],\n",
        "                [indicators_dic[i][1:] for i in indicators_dic],\n",
        "                strict=True,\n",
        "            )\n",
        "        ),\n",
        "        columns=[\"red_flag\", \"R_id\", \"fields needed\"],\n",
        "    )\n",
        "    indicatordf[\"R_id\"] = indicatordf[\"R_id\"].apply(lambda x: \", \".join(map(str, x)))\n",
        "    indicatordf[\"fields needed\"] = indicatordf[\"fields needed\"].astype(str).str.replace(r\"\\[|\\]|'\", \"\", regex=True)\n",
        "    indicatordf[\"calculation\"] = results_list\n",
        "    indicatordf[\"missing fields\"] = missing_fields\n",
        "    indicatordf[\"missing fields\"] = indicatordf[\"missing fields\"].apply(lambda x: \", \".join(map(str, x)))\n",
        "\n",
        "    if check_coverage:\n",
        "        # Calculate coverage\n",
        "        coverage = []\n",
        "        for i in indicators_dic.values():\n",
        "            fields = [item for sublist in i for item in sublist][1:]\n",
        "            result = calculate_coverage(fields, \"release_summary\")\n",
        "            result_value = pd.to_numeric(result[\"total_percentage\"][0])\n",
        "            coverage.append(result_value)\n",
        "        indicatordf[\"coverage\"] = coverage\n",
        "    return indicatordf"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "s75VrpzLRNkx"
      },
      "source": [
        "## Usability analysis"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HTkFw9jajocD"
      },
      "source": [
        "Generate a list of the fields published:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MgZ3TsCOjozN"
      },
      "outputs": [],
      "source": [
        "fields_list = fields_table.iloc[:, 0].tolist()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xrxFlHz9Hk48"
      },
      "outputs": [],
      "source": [
        "indicators_dic = get_red_flags_dictionary(fields_list)\n",
        "result = redflags_checks(fields_list, indicators_dic)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Fvej9yl_H5oE"
      },
      "source": [
        "### Export results"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EBmaz2VRITHf"
      },
      "source": [
        "#### Load use case indicators spreadsheet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "result_final = check_red_flags_indicators(result)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sbcQgxItIodv"
      },
      "source": [
        "#### Table of results"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hMJH_IpAIm9V"
      },
      "outputs": [],
      "source": [
        "result_final"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false
      },
      "source": [
        "#### Results summary"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "table = result_final.groupby(\"calculation\").agg(total_red_flags=(\"R_id\", \"count\")).reset_index()\n",
        "table[\"%\"] = round(table[\"total_red_flags\"] / table[\"total_red_flags\"].sum() * 100, 1)\n",
        "table"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "collapsed": false
      },
      "source": [
        "#### Most common fields to indicators"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "common_fields = most_common_fields_to_calculate_indicators(indicators_dic, fields_table)\n",
        "common_fields"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3001zqj9IxHF"
      },
      "source": [
        "#### Save the table to a spreadsheet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Qxb5b7wsI0GM"
      },
      "outputs": [],
      "source": [
        "set_spreadsheet_name(\"Red Flags\")\n",
        "save_dataframe_to_sheet(result_final, \"red_flags_table\")\n",
        "save_dataframe_to_sheet(common_fields, \"common_fields_table\")\n",
        "save_dataframe_to_sheet(fields_table, \"fields_list\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "name": "template_red_flags_checks_registry",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}