From eac4fbadf3d9e72ebd71c3b1bf102ef802e75f93 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 00:00:46 +0100 Subject: [PATCH 01/18] Refactor - Moved 'settings' to blueprint 'settings' directory --- .../blueprint/settings/__init__.py | 120 +++++++ .../settings/templates/notification-log.html | 19 ++ .../settings/templates/settings.html | 310 ++++++++++++++++++ .../blueprint/tags/templates/edit-tag.html | 2 +- changedetectionio/flask_app.py | 111 +------ .../templates/_common_fields.html | 2 +- changedetectionio/templates/_helpers.html | 2 +- changedetectionio/templates/base.html | 2 +- changedetectionio/templates/edit.html | 2 +- changedetectionio/templates/settings.html | 4 +- .../templates/watch-overview.html | 6 +- .../test_custom_browser_url.py | 2 +- .../tests/fetchers/test_content.py | 2 +- .../tests/proxy_list/test_noproxy.py | 4 +- .../proxy_list/test_select_custom_proxy.py | 2 +- .../tests/proxy_socks5/test_socks5_proxy.py | 2 +- .../proxy_socks5/test_socks5_proxy_sources.py | 2 +- .../tests/restock/test_restock.py | 2 +- .../tests/smtp/test_notification_smtp.py | 4 +- .../tests/test_access_control.py | 16 +- .../tests/test_add_replace_remove_filter.py | 2 +- changedetectionio/tests/test_api.py | 4 +- changedetectionio/tests/test_backend.py | 2 +- .../tests/test_history_consistency.py | 2 +- changedetectionio/tests/test_ignore_text.py | 4 +- .../tests/test_ignorehyperlinks.py | 4 +- .../tests/test_ignorestatuscode.py | 2 +- .../tests/test_ignorewhitespace.py | 2 +- .../tests/test_nonrenderable_pages.py | 4 +- changedetectionio/tests/test_notification.py | 14 +- .../tests/test_notification_errors.py | 2 +- changedetectionio/tests/test_request.py | 6 +- .../tests/test_restock_itemprop.py | 4 +- changedetectionio/tests/test_scheduler.py | 8 +- changedetectionio/tests/test_security.py | 2 +- changedetectionio/tests/util.py | 2 +- 36 files changed, 512 insertions(+), 168 deletions(-) create mode 100644 changedetectionio/blueprint/settings/__init__.py create mode 100644 changedetectionio/blueprint/settings/templates/notification-log.html create mode 100644 changedetectionio/blueprint/settings/templates/settings.html diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py new file mode 100644 index 00000000000..2d876ab0b23 --- /dev/null +++ b/changedetectionio/blueprint/settings/__init__.py @@ -0,0 +1,120 @@ +import os +from copy import deepcopy +from datetime import datetime +from zoneinfo import ZoneInfo, available_timezones +import secrets +import flask_login +from flask import Blueprint, render_template, request, redirect, url_for, flash + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.flask_app import login_optionally_required + + +def construct_blueprint(datastore: ChangeDetectionStore): + settings_blueprint = Blueprint('settings', __name__, template_folder="templates") + + @login_optionally_required + @settings_blueprint.route("/", methods=['GET', "POST"]) + def settings_page(): + from changedetectionio import forms + + default = deepcopy(datastore.data['settings']) + if datastore.proxy_list is not None: + available_proxies = list(datastore.proxy_list.keys()) + # When enabled + system_proxy = datastore.data['settings']['requests']['proxy'] + # In the case it doesnt exist anymore + if not system_proxy in available_proxies: + system_proxy = None + + default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0] + # Used by the form handler to keep or remove the proxy settings + default['proxy_list'] = available_proxies[0] + + # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status + form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, + data=default, + extra_notification_tokens=datastore.get_unique_notification_tokens_available() + ) + + # Remove the last option 'System default' + form.application.form.notification_format.choices.pop() + + if datastore.proxy_list is None: + # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead + del form.requests.form.proxy + else: + form.requests.form.proxy.choices = [] + for p in datastore.proxy_list: + form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) + + if request.method == 'POST': + # Password unset is a GET, but we can lock the session to a salted env password to always need the password + if form.application.form.data.get('removepassword_button', False): + # SALTED_PASS means the password is "locked" to what we set in the Env var + if not os.getenv("SALTED_PASS", False): + datastore.remove_password() + flash("Password protection removed.", 'notice') + flask_login.logout_user() + return redirect(url_for('settings.settings_page')) + + if form.validate(): + # Don't set password to False when a password is set - should be only removed with the `removepassword` button + app_update = dict(deepcopy(form.data['application'])) + + # Never update password with '' or False (Added by wtforms when not in submission) + if 'password' in app_update and not app_update['password']: + del (app_update['password']) + + datastore.data['settings']['application'].update(app_update) + datastore.data['settings']['requests'].update(form.data['requests']) + + if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): + datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password + datastore.needs_write_urgent = True + flash("Password protection enabled.", 'notice') + flask_login.logout_user() + return redirect(url_for('index')) + + datastore.needs_write_urgent = True + flash("Settings updated.") + + else: + flash("An error occurred, please see below.", "error") + + # Convert to ISO 8601 format, all date/time relative events stored as UTC time + utc_time = datetime.now(ZoneInfo("UTC")).isoformat() + + output = render_template("settings.html", + api_key=datastore.data['settings']['application'].get('api_access_token'), + available_timezones=sorted(available_timezones()), + emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), + form=form, + hide_remove_pass=os.getenv("SALTED_PASS", False), + min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), + settings_application=datastore.data['settings']['application'], + timezone_default_config=datastore.data['settings']['application'].get('timezone'), + utc_time=utc_time, + ) + + return output + + @login_optionally_required + @settings_blueprint.route("/reset-api-key", methods=['GET']) + def settings_reset_api_key(): + secret = secrets.token_hex(16) + datastore.data['settings']['application']['api_access_token'] = secret + datastore.needs_write_urgent = True + flash("API Key was regenerated.") + return redirect(url_for('settings.settings_page')+'#api') + + @login_optionally_required + @settings_blueprint.route("/notification-logs", methods=['GET']) + def notification_logs(): + from changedetectionio.flask_app import notification_debug_log + output = render_template("notification-log.html", + logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."]) + return output + + return settings_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/settings/templates/notification-log.html b/changedetectionio/blueprint/settings/templates/notification-log.html new file mode 100644 index 00000000000..ee76e2590be --- /dev/null +++ b/changedetectionio/blueprint/settings/templates/notification-log.html @@ -0,0 +1,19 @@ +{% extends 'base.html' %} + +{% block content %} +
+
+ +

Notification debug log

+
+
    + {% for log in logs|reverse %} +
  • {{log}}
    • + {% endfor %} +
+
+ +
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/settings/templates/settings.html b/changedetectionio/blueprint/settings/templates/settings.html new file mode 100644 index 00000000000..2c044d47780 --- /dev/null +++ b/changedetectionio/blueprint/settings/templates/settings.html @@ -0,0 +1,310 @@ +{% extends 'base.html' %} + +{% block content %} +{% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %} +{% from '_common_fields.html' import render_common_settings_form %} + + + + + + + +
+
+ +
+
+
+ +
+
+
+ {{ render_field(form.requests.form.time_between_check, class="time-check-widget") }} + Default recheck time for all watches, current system minimum is {{min_system_recheck_seconds}} seconds (more info). +
+ +
+ {{ render_time_schedule_form(form.requests, available_timezones, timezone_default_config) }} +
+
+
+
+ {{ render_field(form.requests.form.jitter_seconds, class="jitter_seconds") }} + Example - 3 seconds random jitter could trigger up to 3 seconds earlier or up to 3 seconds later +
+
+ {{ render_field(form.application.form.filter_failure_notification_threshold_attempts, class="filter_failure_notification_threshold_attempts") }} + After this many consecutive times that the CSS/xPath filter is missing, send a notification +
+ Set to 0 to disable + +
+
+ {% if not hide_remove_pass %} + {% if current_user.is_authenticated %} + {{ render_button(form.application.form.removepassword_button) }} + {% else %} + {{ render_field(form.application.form.password) }} + Password protection for your changedetection.io application. + {% endif %} + {% else %} + Password is locked. + {% endif %} +
+ +
+ {{ render_checkbox_field(form.application.form.shared_diff_access, class="shared_diff_access") }} + Allow access to view watch diff page when password is enabled (Good for sharing the diff page) + +
+
+ {{ render_checkbox_field(form.application.form.rss_hide_muted_watches) }} +
+
+ {{ render_field(form.application.form.pager_size) }} + Number of items per page in the watch overview list, 0 to disable. +
+ +
+ {{ render_checkbox_field(form.application.form.extract_title_as_title) }} + Note: This will automatically apply to all existing watches. +
+
+ {{ render_checkbox_field(form.application.form.empty_pages_are_a_change) }} + When a request returns no content, or the HTML does not contain any text, is this considered a change? +
+ {% if form.requests.proxy %} +
+ {{ render_field(form.requests.form.proxy, class="fetch-backend-proxy") }} + + Choose a default proxy for all watches + +
+ {% endif %} +
+
+ +
+
+
+ {{ render_common_settings_form(form.application.form, emailprefix, settings_application, extra_notification_token_placeholder_info) }} +
+
+
+ {{ render_field(form.application.form.base_url, class="m-d") }} + + Base URL used for the {{ '{{ base_url }}' }} token in notification links.
+ Default value is the system environment variable 'BASE_URL' - read more here. + +
+
+ +
+
+ {{ render_field(form.application.form.fetch_backend, class="fetch-backend") }} + +

Use the Basic method (default) where your watched sites don't need Javascript to render.

+

The Chrome/Javascript method requires a network connection to a running WebDriver+Chrome server, set by the ENV var 'WEBDRIVER_URL'.

+ +
+
+
+ If you're having trouble waiting for the page to be fully rendered (text missing etc), try increasing the 'wait' time here. +
+ This will wait n seconds before extracting the text. +
+
+ {{ render_field(form.application.form.webdriver_delay) }} +
+
+
+ {{ render_field(form.requests.form.default_ua) }} + + Applied to all requests.

+ Note: Simply changing the User-Agent often does not defeat anti-robot technologies, it's important to consider all of the ways that the browser is detected. + +
+
+
+ Tip: Connect using Bright Data and Oxylabs Proxies, find out more here. + +
+
+ +
+ +
+ {{ render_checkbox_field(form.application.form.ignore_whitespace) }} + Ignore whitespace, tabs and new-lines/line-feeds when considering if a change was detected.
+ Note: Changing this will change the status of your existing watches, possibly trigger alerts etc. + +
+
+ {{ render_checkbox_field(form.application.form.render_anchor_tag_content) }} + Render anchor tag content, default disabled, when enabled renders links as (link text)[https://somesite.com] +
+ Note: Changing this could affect the content of your existing watches, possibly trigger alerts etc. + +
+
+ {{ render_field(form.application.form.global_subtractive_selectors, rows=5, placeholder="header +footer +nav +.stockticker +//*[contains(text(), 'Advertisement')]") }} + +
    +
  • Remove HTML element(s) by CSS and XPath selectors before text conversion.
    • +
  • Don't paste HTML here, use only CSS and XPath selectors
    • +
  • Add multiple elements, CSS or XPath selectors per line to ignore multiple parts of the HTML.
    • +
+ +
+
+ {{ render_field(form.application.form.global_ignore_text, rows=5, placeholder="Some text to ignore in a line +/some.regex\d{2}/ for case-INsensitive regex + ") }} + Note: This is applied globally in addition to the per-watch rules.
+ +
    +
  • Matching text will be ignored in the text snapshot (you can still see it but it wont trigger a change)
    • +
  • Note: This is applied globally in addition to the per-watch rules.
    • +
  • Each line processed separately, any line matching will be ignored (removed before creating the checksum)
    • +
  • Regular Expression support, wrap the entire line in forward slash /regex/
    • +
  • Changing this will affect the comparison checksum which may trigger an alert
    • +
+ +
+
+ +
+

API Access

+

Drive your changedetection.io via API, More about API access here

+ +
+ {{ render_checkbox_field(form.application.form.api_access_token_enabled) }} +
Restrict API access limit by using x-api-key header - required for the Chrome Extension to work

+

API Key {{api_key}} + +
+
+
+ Regenerate API key +
+
+

Chrome Extension

+

Easily add any web-page to your changedetection.io installation from within Chrome.

+ Step 1 Install the extension, Step 2 Navigate to this page, + Step 3 Open the extension from the toolbar and click "Sync API Access" +

+ + Chrome store icon + Chrome Webstore + +

+
+
+
+
+ Ensure the settings below are correct, they are used to manage the time schedule for checking your web page watches. +
+
+

UTC Time & Date from Server: {{ utc_time }}

+

Local Time & Date in Browser:

+

+ {{ render_field(form.application.form.timezone) }} + +

+
+
+
+ + +

Tip: "Residential" and "Mobile" proxy type can be more successfull than "Data Center" for blocked websites. + +

+ {{ render_field(form.requests.form.extra_proxies) }} + "Name" will be used for selecting the proxy in the Watch Edit settings
+ SOCKS5 proxies with authentication are only supported with 'plain requests' fetcher, for other fetchers you should whitelist the IP access instead +
+
+

+ Extra Browsers can be attached to further defeat CAPTCHA's on websites that are particularly hard to scrape.
+ Simply paste the connection address into the box, More instructions and examples here +

+ {{ render_field(form.requests.form.extra_browsers) }} +
+
+
+
+ {{ render_button(form.save_button) }} + Back + Clear Snapshot History +
+
+
+
+
+ +{% endblock %} diff --git a/changedetectionio/blueprint/tags/templates/edit-tag.html b/changedetectionio/blueprint/tags/templates/edit-tag.html index e527ea52ef1..b8081e3e83f 100644 --- a/changedetectionio/blueprint/tags/templates/edit-tag.html +++ b/changedetectionio/blueprint/tags/templates/edit-tag.html @@ -124,7 +124,7 @@ {% if has_default_notification_urls %}
Look out! - There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications.
{% endif %} Use system defaults diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 6dc9354686b..b1850a028d5 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -924,106 +924,6 @@ def edit_page(uuid): return output - @app.route("/settings", methods=['GET', "POST"]) - @login_optionally_required - def settings_page(): - from changedetectionio import forms - from datetime import datetime - from zoneinfo import available_timezones - - default = deepcopy(datastore.data['settings']) - if datastore.proxy_list is not None: - available_proxies = list(datastore.proxy_list.keys()) - # When enabled - system_proxy = datastore.data['settings']['requests']['proxy'] - # In the case it doesnt exist anymore - if not system_proxy in available_proxies: - system_proxy = None - - default['requests']['proxy'] = system_proxy if system_proxy is not None else available_proxies[0] - # Used by the form handler to keep or remove the proxy settings - default['proxy_list'] = available_proxies[0] - - - # Don't use form.data on POST so that it doesnt overrid the checkbox status from the POST status - form = forms.globalSettingsForm(formdata=request.form if request.method == 'POST' else None, - data=default, - extra_notification_tokens=datastore.get_unique_notification_tokens_available() - ) - - # Remove the last option 'System default' - form.application.form.notification_format.choices.pop() - - if datastore.proxy_list is None: - # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead - del form.requests.form.proxy - else: - form.requests.form.proxy.choices = [] - for p in datastore.proxy_list: - form.requests.form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) - - - if request.method == 'POST': - # Password unset is a GET, but we can lock the session to a salted env password to always need the password - if form.application.form.data.get('removepassword_button', False): - # SALTED_PASS means the password is "locked" to what we set in the Env var - if not os.getenv("SALTED_PASS", False): - datastore.remove_password() - flash("Password protection removed.", 'notice') - flask_login.logout_user() - return redirect(url_for('settings_page')) - - if form.validate(): - # Don't set password to False when a password is set - should be only removed with the `removepassword` button - app_update = dict(deepcopy(form.data['application'])) - - # Never update password with '' or False (Added by wtforms when not in submission) - if 'password' in app_update and not app_update['password']: - del (app_update['password']) - - datastore.data['settings']['application'].update(app_update) - datastore.data['settings']['requests'].update(form.data['requests']) - - if not os.getenv("SALTED_PASS", False) and len(form.application.form.password.encrypted_password): - datastore.data['settings']['application']['password'] = form.application.form.password.encrypted_password - datastore.needs_write_urgent = True - flash("Password protection enabled.", 'notice') - flask_login.logout_user() - return redirect(url_for('index')) - - datastore.needs_write_urgent = True - flash("Settings updated.") - - else: - flash("An error occurred, please see below.", "error") - - # Convert to ISO 8601 format, all date/time relative events stored as UTC time - utc_time = datetime.now(ZoneInfo("UTC")).isoformat() - - output = render_template("settings.html", - api_key=datastore.data['settings']['application'].get('api_access_token'), - available_timezones=sorted(available_timezones()), - emailprefix=os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - extra_notification_token_placeholder_info=datastore.get_unique_notification_token_placeholders_available(), - form=form, - hide_remove_pass=os.getenv("SALTED_PASS", False), - min_system_recheck_seconds=int(os.getenv('MINIMUM_SECONDS_RECHECK_TIME', 3)), - settings_application=datastore.data['settings']['application'], - timezone_default_config=datastore.data['settings']['application'].get('timezone'), - utc_time=utc_time, - ) - - return output - - @app.route("/settings/reset-api-key", methods=['GET']) - @login_optionally_required - def settings_reset_api_key(): - import secrets - secret = secrets.token_hex(16) - datastore.data['settings']['application']['api_access_token'] = secret - datastore.needs_write_urgent = True - flash("API Key was regenerated.") - return redirect(url_for('settings_page')+'#api') @app.route("/import", methods=['GET', "POST"]) @login_optionally_required @@ -1282,14 +1182,6 @@ def preview_page(uuid): return output - @app.route("/settings/notification-logs", methods=['GET']) - @login_optionally_required - def notification_logs(): - global notification_debug_log - output = render_template("notification-log.html", - logs=notification_debug_log if len(notification_debug_log) else ["Notification logs are empty - no notifications sent yet."]) - - return output @app.route("/static//", methods=['GET']) def static_content(group, filename): @@ -1686,6 +1578,9 @@ def highlight_submit_ignore_url(): import changedetectionio.blueprint.backups as backups app.register_blueprint(backups.construct_blueprint(datastore), url_prefix='/backups') + import changedetectionio.blueprint.settings as settings + app.register_blueprint(settings.construct_blueprint(datastore), url_prefix='/settings') + import changedetectionio.conditions.blueprint as conditions app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions') diff --git a/changedetectionio/templates/_common_fields.html b/changedetectionio/templates/_common_fields.html index 9a1cd12853b..ebc27e08924 100644 --- a/changedetectionio/templates/_common_fields.html +++ b/changedetectionio/templates/_common_fields.html @@ -28,7 +28,7 @@ {% if emailprefix %} Add email Add an email address {% endif %} - Notification debug logs + Notification debug logs
diff --git a/changedetectionio/templates/_helpers.html b/changedetectionio/templates/_helpers.html index 461eb22e03a..2ed75a30170 100644 --- a/changedetectionio/templates/_helpers.html +++ b/changedetectionio/templates/_helpers.html @@ -199,7 +199,7 @@ {% else %} - Want to use a time schedule? First confirm/save your Time Zone Settings + Want to use a time schedule? First confirm/save your Time Zone Settings
{% endif %} diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 842e648a304..107c2e5756d 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -64,7 +64,7 @@ GROUPS
  • - SETTINGS + SETTINGS
  • IMPORT diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 92c719670c2..8e30623ebfe 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -274,7 +274,7 @@

    Click here to Start

    {% if has_default_notification_urls %}
    Look out! - There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications. + There are system-wide notification URLs enabled, this form will override notification settings for this watch only ‐ an empty Notification URL list here will still send notifications.
    {% endif %} Use system defaults diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 752ff27e8d8..2c044d47780 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -28,7 +28,7 @@
    -
    +
    @@ -203,7 +203,7 @@

    API Access

    - Regenerate API key + Regenerate API key

    Chrome Extension

    diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 3e248ea2ee5..cd52a41ae04 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -129,9 +129,9 @@ {% if '403' in watch.last_error %} {% if has_proxies %} - Try other proxies/location  + Try other proxies/location  {% endif %} - Try adding external proxies/locations + Try adding external proxies/locations {% endif %} {% if 'empty result or contain only an image' in watch.last_error %} @@ -140,7 +140,7 @@
    {% endif %} {% if watch.last_notification_error is defined and watch.last_notification_error != False %} -
    {{ watch.last_notification_error }}
    +
    {{ watch.last_notification_error }}
    {% endif %} {% if watch['processor'] == 'text_json_diff' %} diff --git a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py index 87490a776c4..ba3c6a1e420 100644 --- a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py +++ b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py @@ -16,7 +16,7 @@ def do_test(client, live_server, make_test_use_extra_browser=False): ##################### res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_webdriver", diff --git a/changedetectionio/tests/fetchers/test_content.py b/changedetectionio/tests/fetchers/test_content.py index 8d468cd4b52..28199186702 100644 --- a/changedetectionio/tests/fetchers/test_content.py +++ b/changedetectionio/tests/fetchers/test_content.py @@ -11,7 +11,7 @@ def test_fetch_webdriver_content(client, live_server, measure_memory_usage): ##################### res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_webdriver"}, diff --git a/changedetectionio/tests/proxy_list/test_noproxy.py b/changedetectionio/tests/proxy_list/test_noproxy.py index f3d0e3003ea..477bdb36adc 100644 --- a/changedetectionio/tests/proxy_list/test_noproxy.py +++ b/changedetectionio/tests/proxy_list/test_noproxy.py @@ -18,7 +18,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage): # Setup a proxy res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", @@ -37,7 +37,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage): # Should be available as an option res = client.get( - url_for("settings_page", unpause_on_save=1)) + url_for("settings.settings_page", unpause_on_save=1)) assert b'No proxy' in res.data diff --git a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py index 266c46e1c13..33cf5d143d4 100644 --- a/changedetectionio/tests/proxy_list/test_select_custom_proxy.py +++ b/changedetectionio/tests/proxy_list/test_select_custom_proxy.py @@ -11,7 +11,7 @@ def test_select_custom(client, live_server, measure_memory_usage): # Goto settings, add our custom one res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py index 56e45f187bb..e59dde95c88 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py @@ -25,7 +25,7 @@ def test_socks5(client, live_server, measure_memory_usage): # Setup a proxy res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py index a5d3b69f617..875544b3002 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy_sources.py @@ -28,7 +28,7 @@ def test_socks5_from_proxiesjson_file(client, live_server, measure_memory_usage) test_url = test_url.replace('localhost.localdomain', 'cdio') test_url = test_url.replace('localhost', 'cdio') - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) assert b'name="requests-proxy" type="radio" value="socks5proxy"' in res.data res = client.post( diff --git a/changedetectionio/tests/restock/test_restock.py b/changedetectionio/tests/restock/test_restock.py index 5a29cc28104..1145fe12fa0 100644 --- a/changedetectionio/tests/restock/test_restock.py +++ b/changedetectionio/tests/restock/test_restock.py @@ -62,7 +62,7 @@ def test_restock_detection(client, live_server, measure_memory_usage): ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title "+default_notification_title, "application-notification_body": "fallback-body "+default_notification_body, diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index 47080a9acd4..fd95ae9f662 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -50,7 +50,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title " + default_notification_title, "application-notification_body": "fallback-body
    " + default_notification_body, @@ -116,7 +116,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title " + default_notification_title, "application-notification_body": notification_body, diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index 67b0592342b..7d93265576f 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -8,7 +8,7 @@ def test_check_access_control(app, client, live_server): with app.test_client(use_cookies=True) as c: # Check we don't have any password protection enabled yet. - res = c.get(url_for("settings_page")) + res = c.get(url_for("settings.settings_page")) assert b"Remove password" not in res.data # add something that we can hit via diff page later @@ -32,7 +32,7 @@ def test_check_access_control(app, client, live_server): # Enable password check and diff page access bypass res = c.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-password": "foobar", "application-shared_diff_access": "True", "requests-time_between_check-minutes": 180, @@ -79,7 +79,7 @@ def test_check_access_control(app, client, live_server): # 598 - Password should be set and not accidently removed res = c.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, @@ -91,7 +91,7 @@ def test_check_access_control(app, client, live_server): assert b"Login" in res.data - res = c.get(url_for("settings_page"), + res = c.get(url_for("settings.settings_page"), follow_redirects=True) @@ -110,7 +110,7 @@ def test_check_access_control(app, client, live_server): # Yes we are correctly logged in assert b"LOG OUT" in res.data - res = c.get(url_for("settings_page")) + res = c.get(url_for("settings.settings_page")) # Menu should be available now assert b"SETTINGS" in res.data @@ -124,7 +124,7 @@ def test_check_access_control(app, client, live_server): # Remove password button, and check that it worked ################################################## res = c.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_webdriver", @@ -139,7 +139,7 @@ def test_check_access_control(app, client, live_server): # Be sure a blank password doesnt setup password protection ############################################################ res = c.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-password": "", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, @@ -151,7 +151,7 @@ def test_check_access_control(app, client, live_server): # Now checking the diff access # Enable password check and diff page access bypass res = c.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-password": "foobar", # Should be disabled # "application-shared_diff_access": "True", diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index 3debbea45e8..ecb1cf81d65 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -111,7 +111,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://') + "?xxx={{ watch_url }}" res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_title": "New ChangeDetection.io Notification - {{ watch_url }}", # triggered_text will contain multiple lines "application-notification_body": 'triggered text was -{{triggered_text}}- ### 网站监测 内容更新了 ####', diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index 6b39822b63d..e975a3dcc7d 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -259,7 +259,7 @@ def test_access_denied(client, live_server, measure_memory_usage): # Disable config_api_token_enabled and it should work res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_requests", @@ -280,7 +280,7 @@ def test_access_denied(client, live_server, measure_memory_usage): assert b'Deleted' in res.data res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_requests", diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index f31f19e0483..a22a0b2355b 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -122,7 +122,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # Enable auto pickup of in settings res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-extract_title_as_title": "1", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, follow_redirects=True diff --git a/changedetectionio/tests/test_history_consistency.py b/changedetectionio/tests/test_history_consistency.py index 7f171c44bfe..9de844c5f92 100644 --- a/changedetectionio/tests/test_history_consistency.py +++ b/changedetectionio/tests/test_history_consistency.py @@ -27,7 +27,7 @@ def test_consistent_history(client, live_server, measure_memory_usage): # Essentially just triggers the DB write/update res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 4a5c86a199b..298c7df24dd 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -172,7 +172,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem # Goto the settings page, add our ignore text res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", @@ -206,7 +206,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem # Check it saved res = client.get( - url_for("settings_page"), + url_for("settings.settings_page"), ) assert bytes(ignore_text.encode('utf-8')) in res.data diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py index d739eb58c3a..fba5bd1ac2e 100644 --- a/changedetectionio/tests/test_ignorehyperlinks.py +++ b/changedetectionio/tests/test_ignorehyperlinks.py @@ -53,7 +53,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag # Goto the settings page, choose to ignore links (dont select/send "application-render_anchor_tag_content") res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-fetch_backend": "html_requests", @@ -90,7 +90,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag # Goto the settings page, ENABLE render anchor tag res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-render_anchor_tag_content": "true", diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index 9ec8086a98d..a5b2da013fb 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -49,7 +49,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me # Goto the settings page, add our ignore text res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_status_codes": "y", diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py index 25d16244e56..688b8139d19 100644 --- a/changedetectionio/tests/test_ignorewhitespace.py +++ b/changedetectionio/tests/test_ignorewhitespace.py @@ -59,7 +59,7 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage): # Goto the settings page, add our ignore text res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "requests-time_between_check-minutes": 180, "application-ignore_whitespace": "y", diff --git a/changedetectionio/tests/test_nonrenderable_pages.py b/changedetectionio/tests/test_nonrenderable_pages.py index ceb4791edde..317667ca478 100644 --- a/changedetectionio/tests/test_nonrenderable_pages.py +++ b/changedetectionio/tests/test_nonrenderable_pages.py @@ -47,7 +47,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure ##################### client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "", # default, OFF, they are NOT a change "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, @@ -78,7 +78,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # ok now do the opposite client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "y", "requests-time_between_check-minutes": 180, 'application-fetch_backend': "html_requests"}, diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index a87e1b7e224..76e5ce9190d 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -28,7 +28,7 @@ def test_check_notification(client, live_server, measure_memory_usage): set_original_response() # Re 360 - new install should have defaults set - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) notification_url = url_for('test_notification_endpoint', _external=True).replace('http', 'json')+"?status_code=204" assert default_notification_body.encode() in res.data @@ -37,7 +37,7 @@ def test_check_notification(client, live_server, measure_memory_usage): ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": notification_url, "application-notification_title": "fallback-title "+default_notification_title, "application-notification_body": "fallback-body "+default_notification_body, @@ -53,7 +53,7 @@ def test_check_notification(client, live_server, measure_memory_usage): env_base_url = os.getenv('BASE_URL', '').strip() if len(env_base_url): logging.debug(">>> BASE_URL enabled, looking for %s", env_base_url) - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) assert bytes(env_base_url.encode('utf-8')) in res.data else: logging.debug(">>> SKIPPING BASE_URL check") @@ -209,7 +209,7 @@ def test_check_notification(client, live_server, measure_memory_usage): wait_for_all_checks(client) assert os.path.exists("test-datastore/notification.txt") == False - res = client.get(url_for("notification_logs")) + res = client.get(url_for("settings.notification_logs")) # be sure we see it in the output log assert b'New ChangeDetection.io Notification - ' + test_url.encode('utf-8') in res.data @@ -294,7 +294,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me test_notification_url = url_for('test_notification_endpoint', _external=True).replace('http://', 'post://')+"?status_code=204&xxx={{ watch_url }}&+custom-header=123&+second=hello+world%20%22space%22" res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "application-fetch_backend": "html_requests", "application-minutes_between_check": 180, @@ -379,7 +379,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage # otherwise other settings would have already existed from previous tests in this file res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "application-fetch_backend": "html_requests", "application-minutes_between_check": 180, @@ -469,7 +469,7 @@ def _test_color_notifications(client, notification_body_token): # otherwise other settings would have already existed from previous tests in this file res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "application-fetch_backend": "html_requests", "application-minutes_between_check": 180, diff --git a/changedetectionio/tests/test_notification_errors.py b/changedetectionio/tests/test_notification_errors.py index 206b8f7ebc7..292ef1c6616 100644 --- a/changedetectionio/tests/test_notification_errors.py +++ b/changedetectionio/tests/test_notification_errors.py @@ -59,7 +59,7 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u # The error should show in the notification logs res = client.get( - url_for("notification_logs")) + url_for("settings.notification_logs")) found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data assert found_name_resolution_error diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index be174a0f574..93714e64750 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -262,7 +262,7 @@ def test_ua_global_override(client, live_server, measure_memory_usage): test_url = url_for('test_headers', _external=True) res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={ "application-fetch_backend": "html_requests", "application-minutes_between_check": 180, @@ -334,13 +334,13 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): form_data["requests-default_ua-html_webdriver"] = webdriver_ua res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data=form_data, follow_redirects=True ) assert b'Settings updated' in res.data - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) # Only when some kind of real browser is setup if os.getenv('PLAYWRIGHT_DRIVER_URL'): diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index e8ed2886a3c..9c9a0e88c1e 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -341,14 +341,14 @@ def test_change_with_notification_values(client, live_server): wait_for_all_checks(client) # Should see new tokens register - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) assert b'{{restock.original_price}}' in res.data assert b'Original price at first check' in res.data ##################### # Set this up for when we remove the notification from the watch, it should fallback with these details res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": notification_url, "application-notification_title": "title new price {{restock.price}}", "application-notification_body": "new price {{restock.price}}", diff --git a/changedetectionio/tests/test_scheduler.py b/changedetectionio/tests/test_scheduler.py index b7978d3d490..24224eb010f 100644 --- a/changedetectionio/tests/test_scheduler.py +++ b/changedetectionio/tests/test_scheduler.py @@ -18,7 +18,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory # The rest of the actual functionality should be covered in the unit-test unit/test_scheduler.py ##################### res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-empty_pages_are_a_change": "", "requests-time_between_check-seconds": 1, "application-timezone": "Pacific/Kiritimati", # Most Forward Time Zone (UTC+14:00) @@ -28,7 +28,7 @@ def test_check_basic_scheduler_functionality(client, live_server, measure_memory assert b"Settings updated." in res.data - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) assert b'Pacific/Kiritimati' in res.data res = client.post( @@ -135,14 +135,14 @@ def test_check_basic_global_scheduler_functionality(client, live_server, measure ##################### res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data=data, follow_redirects=True ) assert b"Settings updated." in res.data - res = client.get(url_for("settings_page")) + res = client.get(url_for("settings.settings_page")) assert b'Pacific/Kiritimati' in res.data wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_security.py b/changedetectionio/tests/test_security.py index 1fa90256d37..d7234637612 100644 --- a/changedetectionio/tests/test_security.py +++ b/changedetectionio/tests/test_security.py @@ -105,7 +105,7 @@ def test_xss(client, live_server, measure_memory_usage): ) # the template helpers were named .jinja which meant they were not having jinja2 autoescape enabled. res = client.post( - url_for("settings_page"), + url_for("settings.settings_page"), data={"application-notification_urls": '"><img src=x onerror=alert(document.domain)>', "application-notification_title": '"><img src=x onerror=alert(document.domain)>', "application-notification_body": '"><img src=x onerror=alert(document.domain)>', diff --git a/changedetectionio/tests/util.py b/changedetectionio/tests/util.py index e90b6aa8af5..2ffe79d1343 100644 --- a/changedetectionio/tests/util.py +++ b/changedetectionio/tests/util.py @@ -100,7 +100,7 @@ def wait_for_notification_endpoint_output(): def extract_api_key_from_UI(client): import re res = client.get( - url_for("settings_page"), + url_for("settings.settings_page"), ) # <span id="api-key">{{api_key}}</span> From a27cd408070b5bb3cc2e6f0ae666c33ed776276b Mon Sep 17 00:00:00 2001 From: dgtlmoon <dgtlmoon@gmail.com> Date: Tue, 18 Mar 2025 00:14:56 +0100 Subject: [PATCH 02/18] Moving RSS to its own blueprint --- changedetectionio/flask_app.py | 90 +------------------ changedetectionio/templates/base.html | 2 +- .../templates/watch-overview.html | 2 +- changedetectionio/tests/test_backend.py | 2 +- changedetectionio/tests/test_group.py | 2 +- changedetectionio/tests/test_rss.py | 4 +- 6 files changed, 10 insertions(+), 92 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index b1850a028d5..b36d0acc7e7 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -168,6 +168,7 @@ def _jinja2_filter_seconds_precise(timestamp): return format(int(time.time()-timestamp), ',d') + # When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object. class User(flask_login.UserMixin): id=None @@ -232,7 +233,6 @@ def decorated_view(*args, **kwargs): return app.login_manager.unauthorized() return func(*args, **kwargs) - return decorated_view def changedetection_app(config=None, datastore_o=None): @@ -340,91 +340,6 @@ def before_request_handle_cookie_x_settings(): return None - @app.route("/rss", methods=['GET']) - def rss(): - now = time.time() - # Always requires token set - app_rss_token = datastore.data['settings']['application'].get('rss_access_token') - rss_url_token = request.args.get('token') - if rss_url_token != app_rss_token: - return "Access denied, bad token", 403 - - from . import diff - limit_tag = request.args.get('tag', '').lower().strip() - # Be sure limit_tag is a uuid - for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): - if limit_tag == tag.get('title', '').lower().strip(): - limit_tag = uuid - - # Sort by last_changed and add the uuid which is usually the key.. - sorted_watches = [] - - # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away - for uuid, watch in datastore.data['watching'].items(): - # @todo tag notification_muted skip also (improve Watch model) - if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'): - continue - if limit_tag and not limit_tag in watch['tags']: - continue - watch['uuid'] = uuid - sorted_watches.append(watch) - - sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) - - fg = FeedGenerator() - fg.title('changedetection.io') - fg.description('Feed description') - fg.link(href='https://changedetection.io') - - for watch in sorted_watches: - - dates = list(watch.history.keys()) - # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected. - if len(dates) < 2: - continue - - if not watch.viewed: - # Re #239 - GUID needs to be individual for each event - # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) - guid = "{}/{}".format(watch['uuid'], watch.last_changed) - fe = fg.add_entry() - - # Include a link to the diff page, they will have to login here to see if password protection is enabled. - # Description is the page you watch, link takes you to the diff JS UI page - # Dict val base_url will get overriden with the env var if it is set. - ext_base_url = datastore.data['settings']['application'].get('active_base_url') - - # Because we are called via whatever web server, flask should figure out the right path ( - diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} - - fe.link(link=diff_link) - - # @todo watch should be a getter - watch.get('title') (internally if URL else..) - - watch_title = watch.get('title') if watch.get('title') else watch.get('url') - fe.title(title=watch_title) - - html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]), - newest_version_file_contents=watch.get_history_snapshot(dates[-1]), - include_equal=False, - line_feed_sep="<br>") - - # @todo Make this configurable and also consider html-colored markup - # @todo User could decide if <link> goes to the diff page, or to the watch link - rss_template = "<html><body>\n<h4><a href=\"{{watch_url}}\">{{watch_title}}</a></h4>\n<p>{{html_diff}}</p>\n</body></html>\n" - content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) - - fe.content(content=content, type='CDATA') - - fe.guid(guid, permalink=False) - dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) - dt = dt.replace(tzinfo=pytz.UTC) - fe.pubDate(dt) - - response = make_response(fg.rss_str()) - response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') - logger.trace(f"RSS generated in {time.time() - now:.3f}s") - return response @app.route("/", methods=['GET']) @login_optionally_required @@ -1584,6 +1499,9 @@ def highlight_submit_ignore_url(): import changedetectionio.conditions.blueprint as conditions app.register_blueprint(conditions.construct_blueprint(datastore), url_prefix='/conditions') + import changedetectionio.blueprint.rss as rss + app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss') + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() threading.Thread(target=notification_runner).start() diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 107c2e5756d..59a834073aa 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -7,7 +7,7 @@ <meta name="description" content="Self hosted website change detection." > <title>Change Detection{{extra_title}} {% if app_rss_token %} - + {% endif %} diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index cd52a41ae04..070c2af778f 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -223,7 +223,7 @@ all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}
  • - RSS Feed + RSS Feed
    • {{ pagination.links }} diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index a22a0b2355b..55a822f11e3 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -80,7 +80,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # #75, and it should be in the RSS feed rss_token = extract_rss_token_from_UI(client) - res = client.get(url_for("rss", token=rss_token, _external=True)) + res = client.get(url_for("rss.feed", token=rss_token, _external=True)) expected_url = url_for('test_endpoint', _external=True) assert b' Date: Tue, 18 Mar 2025 00:25:05 +0100 Subject: [PATCH 03/18] Include modules --- MANIFEST.in | 2 ++ 1 file changed, 2 insertions(+) diff --git a/MANIFEST.in b/MANIFEST.in index eaf04a6f1b3..0c3d7958d6a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,9 +5,11 @@ recursive-include changedetectionio/content_fetchers * recursive-include changedetectionio/conditions * recursive-include changedetectionio/model * recursive-include changedetectionio/processors * +recursive-include changedetectionio/rss * recursive-include changedetectionio/static * recursive-include changedetectionio/templates * recursive-include changedetectionio/tests * +recursive-include changedetectionio/UI * prune changedetectionio/static/package-lock.json prune changedetectionio/static/styles/node_modules prune changedetectionio/static/styles/package-lock.json From cf3dfbd1d80e7982b41dd06e6fb3af00a1742a84 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 00:28:12 +0100 Subject: [PATCH 04/18] forgot modules --- changedetectionio/blueprint/rss/__init__.py | 100 ++++++++ changedetectionio/blueprint/ui/__init__.py | 234 ++++++++++++++++++ .../ui/templates/clear_all_history.html | 49 ++++ 3 files changed, 383 insertions(+) create mode 100644 changedetectionio/blueprint/rss/__init__.py create mode 100644 changedetectionio/blueprint/ui/__init__.py create mode 100644 changedetectionio/blueprint/ui/templates/clear_all_history.html diff --git a/changedetectionio/blueprint/rss/__init__.py b/changedetectionio/blueprint/rss/__init__.py new file mode 100644 index 00000000000..989a548d9af --- /dev/null +++ b/changedetectionio/blueprint/rss/__init__.py @@ -0,0 +1,100 @@ +import time +import datetime +import pytz +from flask import Blueprint, make_response, request, url_for +from loguru import logger +from feedgen.feed import FeedGenerator + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.safe_jinja import render as jinja_render + +def construct_blueprint(datastore: ChangeDetectionStore): + rss_blueprint = Blueprint('rss', __name__) + + @rss_blueprint.route("/", methods=['GET']) + def feed(): + now = time.time() + # Always requires token set + app_rss_token = datastore.data['settings']['application'].get('rss_access_token') + rss_url_token = request.args.get('token') + if rss_url_token != app_rss_token: + return "Access denied, bad token", 403 + + from changedetectionio import diff + limit_tag = request.args.get('tag', '').lower().strip() + # Be sure limit_tag is a uuid + for uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if limit_tag == tag.get('title', '').lower().strip(): + limit_tag = uuid + + # Sort by last_changed and add the uuid which is usually the key.. + sorted_watches = [] + + # @todo needs a .itemsWithTag() or something - then we can use that in Jinaj2 and throw this away + for uuid, watch in datastore.data['watching'].items(): + # @todo tag notification_muted skip also (improve Watch model) + if datastore.data['settings']['application'].get('rss_hide_muted_watches') and watch.get('notification_muted'): + continue + if limit_tag and not limit_tag in watch['tags']: + continue + watch['uuid'] = uuid + sorted_watches.append(watch) + + sorted_watches.sort(key=lambda x: x.last_changed, reverse=False) + + fg = FeedGenerator() + fg.title('changedetection.io') + fg.description('Feed description') + fg.link(href='https://changedetection.io') + + for watch in sorted_watches: + + dates = list(watch.history.keys()) + # Re #521 - Don't bother processing this one if theres less than 2 snapshots, means we never had a change detected. + if len(dates) < 2: + continue + + if not watch.viewed: + # Re #239 - GUID needs to be individual for each event + # @todo In the future make this a configurable link back (see work on BASE_URL https://github.com/dgtlmoon/changedetection.io/pull/228) + guid = "{}/{}".format(watch['uuid'], watch.last_changed) + fe = fg.add_entry() + + # Include a link to the diff page, they will have to login here to see if password protection is enabled. + # Description is the page you watch, link takes you to the diff JS UI page + # Dict val base_url will get overriden with the env var if it is set. + ext_base_url = datastore.data['settings']['application'].get('active_base_url') + + # Because we are called via whatever web server, flask should figure out the right path ( + diff_link = {'href': url_for('diff_history_page', uuid=watch['uuid'], _external=True)} + + fe.link(link=diff_link) + + # @todo watch should be a getter - watch.get('title') (internally if URL else..) + + watch_title = watch.get('title') if watch.get('title') else watch.get('url') + fe.title(title=watch_title) + + html_diff = diff.render_diff(previous_version_file_contents=watch.get_history_snapshot(dates[-2]), + newest_version_file_contents=watch.get_history_snapshot(dates[-1]), + include_equal=False, + line_feed_sep="
    ") + + # @todo Make this configurable and also consider html-colored markup + # @todo User could decide if goes to the diff page, or to the watch link + rss_template = "\n

    {{watch_title}}

    \n

    {{html_diff}}

    \n\n" + content = jinja_render(template_str=rss_template, watch_title=watch_title, html_diff=html_diff, watch_url=watch.link) + + fe.content(content=content, type='CDATA') + + fe.guid(guid, permalink=False) + dt = datetime.datetime.fromtimestamp(int(watch.newest_history_key)) + dt = dt.replace(tzinfo=pytz.UTC) + fe.pubDate(dt) + + response = make_response(fg.rss_str()) + response.headers.set('Content-Type', 'application/rss+xml;charset=utf-8') + logger.trace(f"RSS generated in {time.time() - now:.3f}s") + return response + + return rss_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/__init__.py b/changedetectionio/blueprint/ui/__init__.py new file mode 100644 index 00000000000..b8e0f4cb674 --- /dev/null +++ b/changedetectionio/blueprint/ui/__init__.py @@ -0,0 +1,234 @@ +import time +from flask import Blueprint, request, redirect, url_for, flash, render_template +from loguru import logger +from functools import wraps + +from changedetectionio.store import ChangeDetectionStore + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData): + ui_blueprint = Blueprint('ui', __name__, template_folder="templates") + + # Import this here to avoid circular imports + from changedetectionio.flask_app import login_optionally_required + + @ui_blueprint.route("/clear_history/", methods=['GET']) + @login_optionally_required + def clear_watch_history(uuid): + try: + datastore.clear_watch_history(uuid) + except KeyError: + flash('Watch not found', 'error') + else: + flash("Cleared snapshot history for watch {}".format(uuid)) + + return redirect(url_for('index')) + + @ui_blueprint.route("/clear_history", methods=['GET', 'POST']) + @login_optionally_required + def clear_all_history(): + if request.method == 'POST': + confirmtext = request.form.get('confirmtext') + + if confirmtext == 'clear': + for uuid in datastore.data['watching'].keys(): + datastore.clear_watch_history(uuid) + + flash("Cleared snapshot history for all watches") + else: + flash('Incorrect confirmation text.', 'error') + + return redirect(url_for('index')) + + output = render_template("clear_all_history.html") + return output + + # Clear all statuses, so we do not see the 'unviewed' class + @ui_blueprint.route("/form/mark-all-viewed", methods=['GET']) + @login_optionally_required + def mark_all_viewed(): + # Save the current newest history as the most recently viewed + with_errors = request.args.get('with_errors') == "1" + for watch_uuid, watch in datastore.data['watching'].items(): + if with_errors and not watch.get('last_error'): + continue + datastore.set_last_viewed(watch_uuid, int(time.time())) + + return redirect(url_for('index')) + + @ui_blueprint.route("/api/delete", methods=['GET']) + @login_optionally_required + def form_delete(): + uuid = request.args.get('uuid') + + if uuid != 'all' and not uuid in datastore.data['watching'].keys(): + flash('The watch by UUID {} does not exist.'.format(uuid), 'error') + return redirect(url_for('index')) + + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + datastore.delete(uuid) + flash('Deleted.') + + return redirect(url_for('index')) + + @ui_blueprint.route("/api/clone", methods=['GET']) + @login_optionally_required + def form_clone(): + uuid = request.args.get('uuid') + # More for testing, possible to return the first/only + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + new_uuid = datastore.clone(uuid) + if new_uuid: + if not datastore.data['watching'].get(uuid).get('paused'): + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid})) + flash('Cloned.') + + return redirect(url_for('index')) + + @ui_blueprint.route("/api/checknow", methods=['GET']) + @login_optionally_required + def form_watch_checknow(): + # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True}))) + tag = request.args.get('tag') + uuid = request.args.get('uuid') + with_errors = request.args.get('with_errors') == "1" + + i = 0 + + running_uuids = [] + for t in running_update_threads: + running_uuids.append(t.current_uuid) + + if uuid: + if uuid not in running_uuids: + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + i += 1 + + else: + # Recheck all, including muted + for watch_uuid, watch in datastore.data['watching'].items(): + if not watch['paused']: + if watch_uuid not in running_uuids: + if with_errors and not watch.get('last_error'): + continue + + if tag != None and tag not in watch['tags']: + continue + + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})) + i += 1 + + if i == 1: + flash("Queued 1 watch for rechecking.") + if i > 1: + flash("Queued {} watches for rechecking.".format(i)) + if i == 0: + flash("No watches available to recheck.") + + return redirect(url_for('index')) + + @ui_blueprint.route("/form/checkbox-operations", methods=['POST']) + @login_optionally_required + def form_watch_list_checkbox_operations(): + op = request.form['op'] + uuids = request.form.getlist('uuids') + + if (op == 'delete'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.delete(uuid.strip()) + flash("{} watches deleted".format(len(uuids))) + + elif (op == 'pause'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['paused'] = True + flash("{} watches paused".format(len(uuids))) + + elif (op == 'unpause'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['paused'] = False + flash("{} watches unpaused".format(len(uuids))) + + elif (op == 'mark-viewed'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.set_last_viewed(uuid, int(time.time())) + flash("{} watches updated".format(len(uuids))) + + elif (op == 'mute'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_muted'] = True + flash("{} watches muted".format(len(uuids))) + + elif (op == 'unmute'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_muted'] = False + flash("{} watches un-muted".format(len(uuids))) + + elif (op == 'recheck'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + # Recheck and require a full reprocessing + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + flash("{} watches queued for rechecking".format(len(uuids))) + + elif (op == 'clear-errors'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid]["last_error"] = False + flash(f"{len(uuids)} watches errors cleared") + + elif (op == 'clear-history'): + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.clear_watch_history(uuid) + flash("{} watches cleared/reset.".format(len(uuids))) + + elif (op == 'notification-default'): + from changedetectionio.notification import ( + default_notification_format_for_watch + ) + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + datastore.data['watching'][uuid.strip()]['notification_title'] = None + datastore.data['watching'][uuid.strip()]['notification_body'] = None + datastore.data['watching'][uuid.strip()]['notification_urls'] = [] + datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch + flash("{} watches set to use default notification settings".format(len(uuids))) + + elif (op == 'assign-tag'): + op_extradata = request.form.get('op_extradata', '').strip() + if op_extradata: + tag_uuid = datastore.add_tag(name=op_extradata) + if op_extradata and tag_uuid: + for uuid in uuids: + uuid = uuid.strip() + if datastore.data['watching'].get(uuid): + # Bug in old versions caused by bad edit page/tag handler + if isinstance(datastore.data['watching'][uuid]['tags'], str): + datastore.data['watching'][uuid]['tags'] = [] + + datastore.data['watching'][uuid]['tags'].append(tag_uuid) + + flash(f"{len(uuids)} watches were tagged") + + return redirect(url_for('index')) + + return ui_blueprint \ No newline at end of file diff --git a/changedetectionio/blueprint/ui/templates/clear_all_history.html b/changedetectionio/blueprint/ui/templates/clear_all_history.html new file mode 100644 index 00000000000..fbbaa34f9f2 --- /dev/null +++ b/changedetectionio/blueprint/ui/templates/clear_all_history.html @@ -0,0 +1,49 @@ +{% extends 'base.html' %} {% block content %} +
    +
    + + +
    +
    + This will remove version history (snapshots) for ALL watches, but keep + your list of URLs!
    + You may like to use the BACKUP link first.
    +
    +
    +
    + + + Type in the word clear to confirm that you + understand. +
    +
    +
    + +
    +
    +
    + Cancel +
    +
    + +
    +
    + +{% endblock %} From b1c082f245ce6526528350902f1c27633408ed98 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 00:29:24 +0100 Subject: [PATCH 05/18] Moving basic UI stuff to their own blueprint --- changedetectionio/flask_app.py | 232 ++---------------- .../templates/clear_all_history.html | 2 +- changedetectionio/templates/edit.html | 6 +- changedetectionio/templates/settings.html | 2 +- .../templates/watch-overview.html | 8 +- 5 files changed, 23 insertions(+), 227 deletions(-) diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index b36d0acc7e7..22f160bd179 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -542,39 +542,6 @@ def ajax_callback_send_notification_test(watch_uuid=None): return 'OK - Sent test notifications' - @app.route("/clear_history/", methods=['GET']) - @login_optionally_required - def clear_watch_history(uuid): - try: - datastore.clear_watch_history(uuid) - except KeyError: - flash('Watch not found', 'error') - else: - flash("Cleared snapshot history for watch {}".format(uuid)) - - return redirect(url_for('index')) - - @app.route("/clear_history", methods=['GET', 'POST']) - @login_optionally_required - def clear_all_history(): - - if request.method == 'POST': - confirmtext = request.form.get('confirmtext') - - if confirmtext == 'clear': - changes_removed = 0 - for uuid in datastore.data['watching'].keys(): - datastore.clear_watch_history(uuid) - #TODO: KeyError not checked, as it is above - - flash("Cleared snapshot history for all watches") - else: - flash('Incorrect confirmation text.', 'error') - - return redirect(url_for('index')) - - output = render_template("clear_all_history.html") - return output def _watch_has_tag_options_set(watch): """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" @@ -904,19 +871,6 @@ def import_page(): ) return output - # Clear all statuses, so we do not see the 'unviewed' class - @app.route("/form/mark-all-viewed", methods=['GET']) - @login_optionally_required - def mark_all_viewed(): - - # Save the current newest history as the most recently viewed - with_errors = request.args.get('with_errors') == "1" - for watch_uuid, watch in datastore.data['watching'].items(): - if with_errors and not watch.get('last_error'): - continue - datastore.set_last_viewed(watch_uuid, int(time.time())) - - return redirect(url_for('index')) @app.route("/diff/", methods=['GET', 'POST']) @login_optionally_required @@ -1227,181 +1181,9 @@ def form_quick_watch_add(): - @app.route("/api/delete", methods=['GET']) - @login_optionally_required - def form_delete(): - uuid = request.args.get('uuid') - - if uuid != 'all' and not uuid in datastore.data['watching'].keys(): - flash('The watch by UUID {} does not exist.'.format(uuid), 'error') - return redirect(url_for('index')) - - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - datastore.delete(uuid) - flash('Deleted.') - - return redirect(url_for('index')) - - @app.route("/api/clone", methods=['GET']) - @login_optionally_required - def form_clone(): - uuid = request.args.get('uuid') - # More for testing, possible to return the first/only - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - new_uuid = datastore.clone(uuid) - if new_uuid: - if not datastore.data['watching'].get(uuid).get('paused'): - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=5, item={'uuid': new_uuid})) - flash('Cloned.') - - return redirect(url_for('index')) - - @app.route("/api/checknow", methods=['GET']) - @login_optionally_required - def form_watch_checknow(): - # Forced recheck will skip the 'skip if content is the same' rule (, 'reprocess_existing_data': True}))) - tag = request.args.get('tag') - uuid = request.args.get('uuid') - with_errors = request.args.get('with_errors') == "1" - - i = 0 - - running_uuids = [] - for t in running_update_threads: - running_uuids.append(t.current_uuid) - - if uuid: - if uuid not in running_uuids: - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - i = 1 - - elif tag: - # Items that have this current tag - for watch_uuid, watch in datastore.data['watching'].items(): - if tag in watch.get('tags', {}): - if with_errors and not watch.get('last_error'): - continue - if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: - update_q.put( - queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid}) - ) - i += 1 - else: - # No tag, no uuid, add everything. - for watch_uuid, watch in datastore.data['watching'].items(): - if watch_uuid not in running_uuids and not datastore.data['watching'][watch_uuid]['paused']: - if with_errors and not watch.get('last_error'): - continue - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': watch_uuid})) - i += 1 - flash(f"{i} watches queued for rechecking.") - return redirect(url_for('index', tag=tag)) - @app.route("/form/checkbox-operations", methods=['POST']) - @login_optionally_required - def form_watch_list_checkbox_operations(): - op = request.form['op'] - uuids = request.form.getlist('uuids') - - if (op == 'delete'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.delete(uuid.strip()) - flash("{} watches deleted".format(len(uuids))) - - elif (op == 'pause'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['paused'] = True - flash("{} watches paused".format(len(uuids))) - - elif (op == 'unpause'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['paused'] = False - flash("{} watches unpaused".format(len(uuids))) - - elif (op == 'mark-viewed'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.set_last_viewed(uuid, int(time.time())) - flash("{} watches updated".format(len(uuids))) - - elif (op == 'mute'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_muted'] = True - flash("{} watches muted".format(len(uuids))) - - elif (op == 'unmute'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_muted'] = False - flash("{} watches un-muted".format(len(uuids))) - - elif (op == 'recheck'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - # Recheck and require a full reprocessing - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - flash("{} watches queued for rechecking".format(len(uuids))) - - elif (op == 'clear-errors'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid]["last_error"] = False - flash(f"{len(uuids)} watches errors cleared") - - elif (op == 'clear-history'): - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.clear_watch_history(uuid) - flash("{} watches cleared/reset.".format(len(uuids))) - - elif (op == 'notification-default'): - from changedetectionio.notification import ( - default_notification_format_for_watch - ) - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - datastore.data['watching'][uuid.strip()]['notification_title'] = None - datastore.data['watching'][uuid.strip()]['notification_body'] = None - datastore.data['watching'][uuid.strip()]['notification_urls'] = [] - datastore.data['watching'][uuid.strip()]['notification_format'] = default_notification_format_for_watch - flash("{} watches set to use default notification settings".format(len(uuids))) - - elif (op == 'assign-tag'): - op_extradata = request.form.get('op_extradata', '').strip() - if op_extradata: - tag_uuid = datastore.add_tag(name=op_extradata) - if op_extradata and tag_uuid: - for uuid in uuids: - uuid = uuid.strip() - if datastore.data['watching'].get(uuid): - # Bug in old versions caused by bad edit page/tag handler - if isinstance(datastore.data['watching'][uuid]['tags'], str): - datastore.data['watching'][uuid]['tags'] = [] - - datastore.data['watching'][uuid]['tags'].append(tag_uuid) - - flash(f"{len(uuids)} watches were tagged") - return redirect(url_for('index')) @app.route("/api/share-url", methods=['GET']) @login_optionally_required @@ -1501,6 +1283,20 @@ def highlight_submit_ignore_url(): import changedetectionio.blueprint.rss as rss app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss') + + import changedetectionio.blueprint.ui as ui + app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData), url_prefix='/ui') + + # Route aliases for backward compatibility (especially for tests) + app.add_url_rule('/clear_history/', view_func=lambda uuid: redirect(url_for('ui.clear_watch_history', uuid=uuid)), endpoint='clear_watch_history') + app.add_url_rule('/clear_history', view_func=lambda: redirect(url_for('ui.clear_all_history')), endpoint='clear_all_history') + app.add_url_rule('/form/mark-all-viewed', view_func=lambda: redirect(url_for('ui.mark_all_viewed', **request.args)), endpoint='mark_all_viewed') + app.add_url_rule('/api/delete', view_func=lambda: redirect(url_for('ui.form_delete', **request.args)), endpoint='form_delete') + app.add_url_rule('/api/clone', view_func=lambda: redirect(url_for('ui.form_clone', **request.args)), endpoint='form_clone') + app.add_url_rule('/api/checknow', view_func=lambda: redirect(url_for('ui.form_watch_checknow', **request.args)), endpoint='form_watch_checknow') + app.add_url_rule('/form/checkbox-operations', methods=['POST'], + view_func=lambda: redirect(url_for('ui.form_watch_list_checkbox_operations')), + endpoint='form_watch_list_checkbox_operations') # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() diff --git a/changedetectionio/templates/clear_all_history.html b/changedetectionio/templates/clear_all_history.html index 01433bb02e1..fbbaa34f9f2 100644 --- a/changedetectionio/templates/clear_all_history.html +++ b/changedetectionio/templates/clear_all_history.html @@ -3,7 +3,7 @@
    diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 8e30623ebfe..7b7be0f52fa 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -586,11 +586,11 @@

    Text filtering

    {{ render_button(form.save_button) }} - Delete - Clear History - Create Copy
    diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 2c044d47780..0757867623c 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -300,7 +300,7 @@

    Chrome Extension

    {{ render_button(form.save_button) }} Back - Clear Snapshot History + Clear Snapshot History
    diff --git a/changedetectionio/templates/watch-overview.html b/changedetectionio/templates/watch-overview.html index 070c2af778f..82e62b1bef1 100644 --- a/changedetectionio/templates/watch-overview.html +++ b/changedetectionio/templates/watch-overview.html @@ -25,7 +25,7 @@ Create a shareable link Tip: You can also add 'shared' watches. More info -
    +
    @@ -186,7 +186,7 @@ {% endif %} - {% if watch.uuid in queued_uuids %}Queued{% else %}Recheck{% endif %} Edit {% if watch.history_n >= 2 %} @@ -215,11 +215,11 @@ {% endif %} {% if has_unviewed %}
  • - Mark all viewed + Mark all viewed
    • {% endif %}
  • - Recheck + Recheck all {% if active_tag_uuid %} in "{{active_tag.title}}"{%endif%}
  • From a0daacc4f2bdf2d093bc55ef315eeb4e7e45b948 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 00:48:27 +0100 Subject: [PATCH 06/18] More changes --- changedetectionio/auth_decorator.py | 38 ++++++++++++ changedetectionio/blueprint/rss/__init__.py | 3 + .../blueprint/settings/__init__.py | 8 +-- changedetectionio/blueprint/ui/__init__.py | 4 +- changedetectionio/flask_app.py | 61 +++++++++---------- .../test_custom_browser_url.py | 4 +- .../fetchers/test_custom_js_before_content.py | 2 +- .../tests/proxy_list/test_noproxy.py | 2 +- .../tests/proxy_socks5/test_socks5_proxy.py | 2 +- .../tests/restock/test_restock.py | 4 +- .../tests/smtp/test_notification_smtp.py | 8 +-- .../tests/test_access_control.py | 7 ++- .../tests/test_add_replace_remove_filter.py | 22 +++---- changedetectionio/tests/test_api.py | 4 +- .../test_automatic_follow_ldjson_price.py | 8 +-- changedetectionio/tests/test_backend.py | 18 +++--- .../tests/test_block_while_text_present.py | 10 +-- changedetectionio/tests/test_clone.py | 2 +- changedetectionio/tests/test_conditions.py | 10 +-- changedetectionio/tests/test_css_selector.py | 4 +- .../tests/test_element_removal.py | 10 +-- changedetectionio/tests/test_errorhandling.py | 8 +-- changedetectionio/tests/test_extract_csv.py | 2 +- changedetectionio/tests/test_extract_regex.py | 4 +- .../tests/test_filter_exist_changes.py | 2 +- .../tests/test_filter_failure_notification.py | 10 +-- changedetectionio/tests/test_group.py | 24 ++++---- changedetectionio/tests/test_ignore_text.py | 16 ++--- .../tests/test_ignorehyperlinks.py | 8 +-- .../tests/test_ignorestatuscode.py | 4 +- .../tests/test_ignorewhitespace.py | 4 +- changedetectionio/tests/test_import.py | 12 ++-- .../tests/test_jsonpath_jq_selector.py | 22 +++---- changedetectionio/tests/test_live_preview.py | 2 +- .../tests/test_nonrenderable_pages.py | 10 +-- changedetectionio/tests/test_notification.py | 26 ++++---- .../tests/test_notification_errors.py | 2 +- changedetectionio/tests/test_pdf.py | 4 +- .../tests/test_preview_endpoints.py | 4 +- changedetectionio/tests/test_request.py | 12 ++-- .../tests/test_restock_itemprop.py | 54 ++++++++-------- changedetectionio/tests/test_rss.py | 8 +-- changedetectionio/tests/test_scheduler.py | 4 +- changedetectionio/tests/test_security.py | 2 +- changedetectionio/tests/test_share_watch.py | 2 +- changedetectionio/tests/test_source.py | 4 +- changedetectionio/tests/test_trigger.py | 8 +-- changedetectionio/tests/test_trigger_regex.py | 6 +- .../tests/test_trigger_regex_with_filter.py | 6 +- changedetectionio/tests/test_unique_lines.py | 14 ++--- .../tests/test_xpath_selector.py | 28 ++++----- .../tests/visualselector/test_fetch_data.py | 4 +- 52 files changed, 292 insertions(+), 255 deletions(-) create mode 100644 changedetectionio/auth_decorator.py diff --git a/changedetectionio/auth_decorator.py b/changedetectionio/auth_decorator.py new file mode 100644 index 00000000000..9121260e39d --- /dev/null +++ b/changedetectionio/auth_decorator.py @@ -0,0 +1,38 @@ +import os +from functools import wraps +from flask import current_app, redirect, request +from loguru import logger + +def login_optionally_required(func): + """ + If password authentication is enabled, verify the user is logged in. + To be used as a decorator for routes that should optionally require login. + This version is blueprint-friendly as it uses current_app instead of directly accessing app. + """ + logger.debug(f"login_optionally_required being applied to {func.__name__}") + @wraps(func) + def decorated_view(*args, **kwargs): + logger.debug(f"login_optionally_required.decorated_view called for {func.__name__}, endpoint: {request.endpoint}") + from flask import current_app + import flask_login + from flask_login import current_user + + # Access datastore through the app config + datastore = current_app.config['DATASTORE'] + has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) + + # Permitted + if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles': + return func(*args, **kwargs) + # Permitted + elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'): + return func(*args, **kwargs) + elif request.method in flask_login.config.EXEMPT_METHODS: + return func(*args, **kwargs) + elif current_app.config.get('LOGIN_DISABLED'): + return func(*args, **kwargs) + elif has_password_enabled and not current_user.is_authenticated: + return current_app.login_manager.unauthorized() + + return func(*args, **kwargs) + return decorated_view \ No newline at end of file diff --git a/changedetectionio/blueprint/rss/__init__.py b/changedetectionio/blueprint/rss/__init__.py index 989a548d9af..93ed09f9aea 100644 --- a/changedetectionio/blueprint/rss/__init__.py +++ b/changedetectionio/blueprint/rss/__init__.py @@ -10,6 +10,9 @@ def construct_blueprint(datastore: ChangeDetectionStore): rss_blueprint = Blueprint('rss', __name__) + + # Import the login decorator if needed + # from changedetectionio.auth_decorator import login_optionally_required @rss_blueprint.route("/", methods=['GET']) def feed(): diff --git a/changedetectionio/blueprint/settings/__init__.py b/changedetectionio/blueprint/settings/__init__.py index 2d876ab0b23..5375b565cc0 100644 --- a/changedetectionio/blueprint/settings/__init__.py +++ b/changedetectionio/blueprint/settings/__init__.py @@ -7,14 +7,14 @@ from flask import Blueprint, render_template, request, redirect, url_for, flash from changedetectionio.store import ChangeDetectionStore -from changedetectionio.flask_app import login_optionally_required +from changedetectionio.auth_decorator import login_optionally_required def construct_blueprint(datastore: ChangeDetectionStore): settings_blueprint = Blueprint('settings', __name__, template_folder="templates") - @login_optionally_required @settings_blueprint.route("/", methods=['GET', "POST"]) + @login_optionally_required def settings_page(): from changedetectionio import forms @@ -100,8 +100,8 @@ def settings_page(): return output - @login_optionally_required @settings_blueprint.route("/reset-api-key", methods=['GET']) + @login_optionally_required def settings_reset_api_key(): secret = secrets.token_hex(16) datastore.data['settings']['application']['api_access_token'] = secret @@ -109,8 +109,8 @@ def settings_reset_api_key(): flash("API Key was regenerated.") return redirect(url_for('settings.settings_page')+'#api') - @login_optionally_required @settings_blueprint.route("/notification-logs", methods=['GET']) + @login_optionally_required def notification_logs(): from changedetectionio.flask_app import notification_debug_log output = render_template("notification-log.html", diff --git a/changedetectionio/blueprint/ui/__init__.py b/changedetectionio/blueprint/ui/__init__.py index b8e0f4cb674..19790ac78ac 100644 --- a/changedetectionio/blueprint/ui/__init__.py +++ b/changedetectionio/blueprint/ui/__init__.py @@ -8,8 +8,8 @@ def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData): ui_blueprint = Blueprint('ui', __name__, template_folder="templates") - # Import this here to avoid circular imports - from changedetectionio.flask_app import login_optionally_required + # Import the login decorator + from changedetectionio.auth_decorator import login_optionally_required @ui_blueprint.route("/clear_history/", methods=['GET']) @login_optionally_required diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 22f160bd179..023aa35d377 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -168,6 +168,8 @@ def _jinja2_filter_seconds_precise(timestamp): return format(int(time.time()-timestamp), ',d') +# Import login_optionally_required from auth_decorator +from changedetectionio.auth_decorator import login_optionally_required # When nobody is logged in Flask-Login's current_user is set to an AnonymousUser object. class User(flask_login.UserMixin): @@ -213,27 +215,6 @@ def check_password(self, password): pass -def login_optionally_required(func): - @wraps(func) - def decorated_view(*args, **kwargs): - - has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) - - # Permitted - if request.endpoint == 'static_content' and request.view_args['group'] == 'styles': - return func(*args, **kwargs) - # Permitted - elif request.endpoint == 'diff_history_page' and datastore.data['settings']['application'].get('shared_diff_access'): - return func(*args, **kwargs) - elif request.method in flask_login.config.EXEMPT_METHODS: - return func(*args, **kwargs) - elif app.config.get('LOGIN_DISABLED'): - return func(*args, **kwargs) - elif has_password_enabled and not current_user.is_authenticated: - return app.login_manager.unauthorized() - - return func(*args, **kwargs) - return decorated_view def changedetection_app(config=None, datastore_o=None): logger.trace("TRACE log is enabled") @@ -248,6 +229,30 @@ def changedetection_app(config=None, datastore_o=None): login_manager = flask_login.LoginManager(app) login_manager.login_view = 'login' app.secret_key = init_app_secret(config['datastore_path']) + + # Set up a request hook to check authentication for all routes + @app.before_request + def check_authentication(): + has_password_enabled = datastore.data['settings']['application'].get('password') or os.getenv("SALTED_PASS", False) + + if has_password_enabled and not flask_login.current_user.is_authenticated: + # Permitted + if request.endpoint and 'static_content' in request.endpoint and request.view_args and request.view_args.get('group') == 'styles': + return None + # Permitted + elif request.endpoint and 'login' in request.endpoint: + return None + elif request.endpoint and 'diff_history_page' in request.endpoint and datastore.data['settings']['application'].get('shared_diff_access'): + return None + elif request.method in flask_login.config.EXEMPT_METHODS: + return None + elif app.config.get('LOGIN_DISABLED'): + return None + # RSS access with token is allowed + elif request.endpoint and 'rss.feed' in request.endpoint: + return None + else: + return login_manager.unauthorized() watch_api.add_resource(api_v1.WatchSingleHistory, @@ -1285,18 +1290,8 @@ def highlight_submit_ignore_url(): app.register_blueprint(rss.construct_blueprint(datastore), url_prefix='/rss') import changedetectionio.blueprint.ui as ui - app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData), url_prefix='/ui') - - # Route aliases for backward compatibility (especially for tests) - app.add_url_rule('/clear_history/', view_func=lambda uuid: redirect(url_for('ui.clear_watch_history', uuid=uuid)), endpoint='clear_watch_history') - app.add_url_rule('/clear_history', view_func=lambda: redirect(url_for('ui.clear_all_history')), endpoint='clear_all_history') - app.add_url_rule('/form/mark-all-viewed', view_func=lambda: redirect(url_for('ui.mark_all_viewed', **request.args)), endpoint='mark_all_viewed') - app.add_url_rule('/api/delete', view_func=lambda: redirect(url_for('ui.form_delete', **request.args)), endpoint='form_delete') - app.add_url_rule('/api/clone', view_func=lambda: redirect(url_for('ui.form_clone', **request.args)), endpoint='form_clone') - app.add_url_rule('/api/checknow', view_func=lambda: redirect(url_for('ui.form_watch_checknow', **request.args)), endpoint='form_watch_checknow') - app.add_url_rule('/form/checkbox-operations', methods=['POST'], - view_func=lambda: redirect(url_for('ui.form_watch_list_checkbox_operations')), - endpoint='form_watch_list_checkbox_operations') + app.register_blueprint(ui.construct_blueprint(datastore, update_q, running_update_threads, queuedWatchMetaData)) + # @todo handle ctrl break ticker_thread = threading.Thread(target=ticker_thread_check_time_launch_checks).start() diff --git a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py index ba3c6a1e420..cc713064d74 100644 --- a/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py +++ b/changedetectionio/tests/custom_browser_url/test_custom_browser_url.py @@ -64,8 +64,8 @@ def do_test(client, live_server, make_test_use_extra_browser=False): wait_for_all_checks(client) # Force recheck - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) diff --git a/changedetectionio/tests/fetchers/test_custom_js_before_content.py b/changedetectionio/tests/fetchers/test_custom_js_before_content.py index d92694a0055..b1556f0700b 100644 --- a/changedetectionio/tests/fetchers/test_custom_js_before_content.py +++ b/changedetectionio/tests/fetchers/test_custom_js_before_content.py @@ -51,6 +51,6 @@ def test_execute_custom_js(client, live_server, measure_memory_usage): assert b"user-agent: mycustomagent" in res.data client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) \ No newline at end of file diff --git a/changedetectionio/tests/proxy_list/test_noproxy.py b/changedetectionio/tests/proxy_list/test_noproxy.py index 477bdb36adc..e1281bd10b2 100644 --- a/changedetectionio/tests/proxy_list/test_noproxy.py +++ b/changedetectionio/tests/proxy_list/test_noproxy.py @@ -67,7 +67,7 @@ def test_noproxy_option(client, live_server, measure_memory_usage): ) assert b"unpaused" in res.data wait_for_all_checks(client) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # Now the request should NOT appear in the second-squid logs (handled by the run_test_proxies.sh script) diff --git a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py index e59dde95c88..61905254f2c 100644 --- a/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py +++ b/changedetectionio/tests/proxy_socks5/test_socks5_proxy.py @@ -97,6 +97,6 @@ def test_socks5(client, live_server, measure_memory_usage): ) assert b"OK" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/restock/test_restock.py b/changedetectionio/tests/restock/test_restock.py index 1145fe12fa0..7b66b900891 100644 --- a/changedetectionio/tests/restock/test_restock.py +++ b/changedetectionio/tests/restock/test_restock.py @@ -88,7 +88,7 @@ def test_restock_detection(client, live_server, measure_memory_usage): # Is it correctly shown as in stock set_back_in_stock_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'not-in-stock' not in res.data @@ -101,7 +101,7 @@ def test_restock_detection(client, live_server, measure_memory_usage): # Default behaviour is to only fire notification when it goes OUT OF STOCK -> IN STOCK # So here there should be no file, because we go IN STOCK -> OUT OF STOCK set_original_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(5) assert not os.path.isfile("test-datastore/notification.txt"), "No notification should have fired when it went OUT OF STOCK by default" diff --git a/changedetectionio/tests/smtp/test_notification_smtp.py b/changedetectionio/tests/smtp/test_notification_smtp.py index fd95ae9f662..5fb750434ff 100644 --- a/changedetectionio/tests/smtp/test_notification_smtp.py +++ b/changedetectionio/tests/smtp/test_notification_smtp.py @@ -75,7 +75,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas set_longer_modified_response() time.sleep(2) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(3) @@ -88,7 +88,7 @@ def test_check_notification_email_formats_default_HTML(client, live_server, meas assert '(added) So let\'s see what happens.\r\n' in msg # The plaintext part with \r\n assert 'Content-Type: text/html' in msg assert '(added) So let\'s see what happens.
    ' in msg # the html part - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -140,7 +140,7 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv wait_for_all_checks(client) set_longer_modified_response() time.sleep(2) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(3) @@ -182,5 +182,5 @@ def test_check_notification_email_formats_default_Text_override_HTML(client, liv assert '<' not in msg assert 'Content-Type: text/html' in msg - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_access_control.py b/changedetectionio/tests/test_access_control.py index 7d93265576f..7730349c172 100644 --- a/changedetectionio/tests/test_access_control.py +++ b/changedetectionio/tests/test_access_control.py @@ -1,4 +1,4 @@ -from .util import live_server_setup, extract_UUID_from_client, wait_for_all_checks +from .util import live_server_setup from flask import url_for import time @@ -23,8 +23,9 @@ def test_check_access_control(app, client, live_server): # causes a 'Popped wrong request context.' error when client. is accessed? #wait_for_all_checks(client) - res = c.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = c.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data + time.sleep(3) # causes a 'Popped wrong request context.' error when client. is accessed? #wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_add_replace_remove_filter.py b/changedetectionio/tests/test_add_replace_remove_filter.py index ecb1cf81d65..f4d874b235e 100644 --- a/changedetectionio/tests/test_add_replace_remove_filter.py +++ b/changedetectionio/tests/test_add_replace_remove_filter.py @@ -69,8 +69,8 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory set_original(excluding='Something irrelevant') # A line thats not the trigger should not trigger anything - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data @@ -79,28 +79,28 @@ def test_check_removed_line_contains_trigger(client, live_server, measure_memory set_original(excluding='The golden line') # Check in the processor here what's going on, its triggering empty-reply and no change. - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data # Now add it back, and we should not get a trigger - client.get(url_for("mark_all_viewed"), follow_redirects=True) + client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) set_original(excluding=None) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data # Remove it again, and we should get a trigger set_original(excluding='The golden line') - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -153,8 +153,8 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa set_original(excluding='Something irrelevant') # A line thats not the trigger should not trigger anything - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) res = client.get(url_for("index")) @@ -162,7 +162,7 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa # The trigger line is ADDED, this should trigger set_original(add_line='

    Oh yes please

    ') - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) @@ -176,5 +176,5 @@ def test_check_add_line_contains_trigger(client, live_server, measure_memory_usa assert b'-Oh yes please' in response assert '网站监测 内容更新了'.encode('utf-8') in response - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_api.py b/changedetectionio/tests/test_api.py index e975a3dcc7d..aa1a42237c1 100644 --- a/changedetectionio/tests/test_api.py +++ b/changedetectionio/tests/test_api.py @@ -276,7 +276,7 @@ def test_access_denied(client, live_server, measure_memory_usage): assert res.status_code == 200 # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data res = client.post( @@ -368,7 +368,7 @@ def test_api_watch_PUT_update(client, live_server, measure_memory_usage): assert b'Additional properties are not allowed' in res.data # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_automatic_follow_ldjson_price.py b/changedetectionio/tests/test_automatic_follow_ldjson_price.py index 46abb2e4f46..37e75b01389 100644 --- a/changedetectionio/tests/test_automatic_follow_ldjson_price.py +++ b/changedetectionio/tests/test_automatic_follow_ldjson_price.py @@ -102,7 +102,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) #time.sleep(1) client.get(url_for('price_data_follower.accept', uuid=uuid, follow_redirects=True)) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # Offer should be gone res = client.get(url_for("index")) @@ -121,7 +121,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage # And not this cause its not the ld-json assert b"So let's see what happens" not in res.data - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) ########################################################################################## # And we shouldnt see the offer @@ -140,7 +140,7 @@ def test_check_ldjson_price_autodetect(client, live_server, measure_memory_usage assert b'ldjson-price-track-offer' not in res.data ########################################################################################## - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_data): @@ -160,7 +160,7 @@ def _test_runner_check_bad_format_ignored(live_server, client, has_ldjson_price_ ########################################################################################## - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_bad_ldjson_is_correctly_ignored(client, live_server, measure_memory_usage): diff --git a/changedetectionio/tests/test_backend.py b/changedetectionio/tests/test_backend.py index 55a822f11e3..c748b8cd06b 100644 --- a/changedetectionio/tests/test_backend.py +++ b/changedetectionio/tests/test_backend.py @@ -33,7 +33,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # Do this a few times.. ensures we dont accidently set the status for n in range(3): - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -63,8 +63,8 @@ def test_check_basic_change_detection_functionality(client, live_server, measure set_modified_response() # Force recheck - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) @@ -106,7 +106,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # Do this a few times.. ensures we dont accidently set the status for n in range(2): - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -128,7 +128,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure follow_redirects=True ) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) @@ -142,19 +142,19 @@ def test_check_basic_change_detection_functionality(client, live_server, measure time.sleep(1) # hit the mark all viewed link - res = client.get(url_for("mark_all_viewed"), follow_redirects=True) + res = client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) assert b'Mark all viewed' not in res.data assert b'unviewed' not in res.data # #2458 "clear history" should make the Watch object update its status correctly when the first snapshot lands again - client.get(url_for("clear_watch_history", uuid=uuid)) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.clear_watch_history", uuid=uuid)) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'preview/' in res.data # # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_block_while_text_present.py b/changedetectionio/tests/test_block_while_text_present.py index 7b5dec9c1a1..a8c72bbf481 100644 --- a/changedetectionio/tests/test_block_while_text_present.py +++ b/changedetectionio/tests/test_block_while_text_present.py @@ -101,7 +101,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu assert bytes(ignore_text.encode('utf-8')) in res.data # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -115,7 +115,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu set_modified_original_ignore_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -127,7 +127,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu # 2548 # Going back to the ORIGINAL should NOT trigger a change set_original_ignore_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data @@ -135,7 +135,7 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu # Now we set a change where the text is gone AND its different content, it should now trigger set_modified_response_minus_block_text() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data @@ -143,5 +143,5 @@ def test_check_block_changedetection_text_NOT_present(client, live_server, measu - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_clone.py b/changedetectionio/tests/test_clone.py index 2e97c77e666..6dbfb33aae7 100644 --- a/changedetectionio/tests/test_clone.py +++ b/changedetectionio/tests/test_clone.py @@ -23,7 +23,7 @@ def test_trigger_functionality(client, live_server, measure_memory_usage): res = client.get( - url_for("form_clone", uuid="first"), + url_for("ui.form_clone", uuid="first"), follow_redirects=True ) diff --git a/changedetectionio/tests/test_conditions.py b/changedetectionio/tests/test_conditions.py index ad11884c583..59a627272ae 100644 --- a/changedetectionio/tests/test_conditions.py +++ b/changedetectionio/tests/test_conditions.py @@ -103,12 +103,12 @@ def test_conditions_with_text_and_number(client, live_server): assert b"Updated watch." in res.data wait_for_all_checks(client) - client.get(url_for("mark_all_viewed"), follow_redirects=True) + client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) wait_for_all_checks(client) # Case 1 set_number_in_range_response("70.5") - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # 75 is > 20 and < 100 and contains "5" @@ -118,16 +118,16 @@ def test_conditions_with_text_and_number(client, live_server): # Case 2: Change with one condition violated # Number out of range (150) but contains '5' - client.get(url_for("mark_all_viewed"), follow_redirects=True) + client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) set_number_out_of_range_response("150.5") - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # Should NOT be marked as having changes since not all conditions are met res = client.get(url_for("index")) assert b'unviewed' not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_css_selector.py b/changedetectionio/tests/test_css_selector.py index 999ea729043..0d1717ec29a 100644 --- a/changedetectionio/tests/test_css_selector.py +++ b/changedetectionio/tests/test_css_selector.py @@ -113,7 +113,7 @@ def test_check_markup_include_filters_restriction(client, live_server, measure_m set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) @@ -236,7 +236,7 @@ def test_filter_is_empty_help_suggestion(client, live_server, measure_memory_usa """) - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get( diff --git a/changedetectionio/tests/test_element_removal.py b/changedetectionio/tests/test_element_removal.py index 3dc510338a6..6663d50ead5 100644 --- a/changedetectionio/tests/test_element_removal.py +++ b/changedetectionio/tests/test_element_removal.py @@ -185,8 +185,8 @@ def test_element_removal_full(client, live_server, measure_memory_usage): assert bytes(subtractive_selectors_data.encode("utf-8")) in res.data # Trigger a check - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) @@ -197,8 +197,8 @@ def test_element_removal_full(client, live_server, measure_memory_usage): set_modified_response() # Trigger a check - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data # Give the thread time to pick it up wait_for_all_checks(client) @@ -228,7 +228,7 @@ def test_element_removal_nth_offset_no_shift(client, live_server, measure_memory for selector_list in subtractive_selectors_data: - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # Add our URL to the import page diff --git a/changedetectionio/tests/test_errorhandling.py b/changedetectionio/tests/test_errorhandling.py index 0cc159b8114..528542f1bbf 100644 --- a/changedetectionio/tests/test_errorhandling.py +++ b/changedetectionio/tests/test_errorhandling.py @@ -50,7 +50,7 @@ def _runner_test_http_errors(client, live_server, http_code, expected_text): #assert b'Error Screenshot' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -59,7 +59,7 @@ def test_http_error_handler(client, live_server, measure_memory_usage): _runner_test_http_errors(client, live_server, 404, 'Page not found') _runner_test_http_errors(client, live_server, 500, '(Internal server error) received') _runner_test_http_errors(client, live_server, 400, 'Error - Request returned a HTTP error code 400') - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # Just to be sure error text is properly handled @@ -83,7 +83,7 @@ def test_DNS_errors(client, live_server, measure_memory_usage): assert found_name_resolution_error # Should always record that we tried assert bytes("just now".encode('utf-8')) in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # Re 1513 @@ -126,5 +126,5 @@ def test_low_level_errors_clear_correctly(client, live_server, measure_memory_us found_name_resolution_error = b"Temporary failure in name resolution" in res.data or b"Name or service not known" in res.data assert not found_name_resolution_error - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_extract_csv.py b/changedetectionio/tests/test_extract_csv.py index 4616679d160..9f83417a0d1 100644 --- a/changedetectionio/tests/test_extract_csv.py +++ b/changedetectionio/tests/test_extract_csv.py @@ -36,7 +36,7 @@ def test_check_extract_text_from_diff(client, live_server, measure_memory_usage) with open("test-datastore/endpoint-content.txt", "w") as f: f.write("Now it's {} seconds since epoch, time flies!".format(last_date)) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.post( diff --git a/changedetectionio/tests/test_extract_regex.py b/changedetectionio/tests/test_extract_regex.py index 058b3411731..09419721e3f 100644 --- a/changedetectionio/tests/test_extract_regex.py +++ b/changedetectionio/tests/test_extract_regex.py @@ -168,7 +168,7 @@ def test_check_filter_and_regex_extract(client, live_server, measure_memory_usag set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -228,5 +228,5 @@ def test_regex_error_handling(client, live_server, measure_memory_usage): assert b'is not a valid regular expression.' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_filter_exist_changes.py b/changedetectionio/tests/test_filter_exist_changes.py index 49d9ff9b459..c344ccea11f 100644 --- a/changedetectionio/tests/test_filter_exist_changes.py +++ b/changedetectionio/tests/test_filter_exist_changes.py @@ -108,7 +108,7 @@ def test_filter_doesnt_exist_then_exists_should_get_notification(client, live_se assert not os.path.isfile("test-datastore/notification.txt") # Now the filter should exist set_response_with_filter() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_notification_endpoint_output() diff --git a/changedetectionio/tests/test_filter_failure_notification.py b/changedetectionio/tests/test_filter_failure_notification.py index 0b0304e2752..330c8aef76e 100644 --- a/changedetectionio/tests/test_filter_failure_notification.py +++ b/changedetectionio/tests/test_filter_failure_notification.py @@ -36,7 +36,7 @@ def run_filter_test(client, live_server, content_filter): # cleanup for the next client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) if os.path.isfile("test-datastore/notification.txt"): @@ -111,7 +111,7 @@ def run_filter_test(client, live_server, content_filter): ATTEMPT_THRESHOLD_SETTING = live_server.app.config['DATASTORE'].data['settings']['application'].get('filter_failure_notification_threshold_attempts', 0) for i in range(0, ATTEMPT_THRESHOLD_SETTING - 2): checked += 1 - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'Warning, no filters were found' in res.data @@ -122,7 +122,7 @@ def run_filter_test(client, live_server, content_filter): time.sleep(2) # One more check should trigger the _FILTER_FAILURE_THRESHOLD_ATTEMPTS_DEFAULT threshold - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) wait_for_notification_endpoint_output() @@ -141,7 +141,7 @@ def run_filter_test(client, live_server, content_filter): # Try several times, it should NOT have 'filter not found' for i in range(0, ATTEMPT_THRESHOLD_SETTING + 2): - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) wait_for_notification_endpoint_output() @@ -157,7 +157,7 @@ def run_filter_test(client, live_server, content_filter): # cleanup for the next client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) os.unlink("test-datastore/notification.txt") diff --git a/changedetectionio/tests/test_group.py b/changedetectionio/tests/test_group.py index 376dbc78580..453701c3bb4 100644 --- a/changedetectionio/tests/test_group.py +++ b/changedetectionio/tests/test_group.py @@ -117,7 +117,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage): assert b"1 Imported" in res.data wait_for_all_checks(client) set_modified_response() - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) rss_token = extract_rss_token_from_UI(client) res = client.get( @@ -127,7 +127,7 @@ def test_setup_group_tag(client, live_server, measure_memory_usage): assert b"should-be-excluded" not in res.data assert res.status_code == 200 assert b"first-imported=1" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_tag_import_singular(client, live_server, measure_memory_usage): @@ -147,7 +147,7 @@ def test_tag_import_singular(client, live_server, measure_memory_usage): ) # Should be only 1 tag because they both had the same assert res.data.count(b'test-tag') == 1 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_tag_add_in_ui(client, live_server, measure_memory_usage): @@ -164,7 +164,7 @@ def test_tag_add_in_ui(client, live_server, measure_memory_usage): res = client.get(url_for("tags.delete_all"), follow_redirects=True) assert b'All tags deleted' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_group_tag_notification(client, live_server, measure_memory_usage): @@ -211,7 +211,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage): wait_for_all_checks(client) set_modified_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(3) assert os.path.isfile("test-datastore/notification.txt") @@ -232,7 +232,7 @@ def test_group_tag_notification(client, live_server, measure_memory_usage): #@todo Test that multiple notifications fired #@todo Test that each of multiple notifications with different settings - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_limit_tag_ui(client, live_server, measure_memory_usage): @@ -269,7 +269,7 @@ def test_limit_tag_ui(client, live_server, measure_memory_usage): assert b'test-tag' in res.data assert res.data.count(b'processor-text_json_diff') == 20 assert b"object at" not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data res = client.get(url_for("tags.delete_all"), follow_redirects=True) assert b'All tags deleted' in res.data @@ -289,13 +289,13 @@ def test_clone_tag_on_import(client, live_server, measure_memory_usage): assert b'another-tag' in res.data watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) - res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True) + res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True) assert b'Cloned' in res.data # 2 times plus the top link to tag assert res.data.count(b'test-tag') == 3 assert res.data.count(b'another-tag') == 3 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usage): @@ -316,13 +316,13 @@ def test_clone_tag_on_quickwatchform_add(client, live_server, measure_memory_usa assert b'another-tag' in res.data watch_uuid = next(iter(live_server.app.config['DATASTORE'].data['watching'])) - res = client.get(url_for("form_clone", uuid=watch_uuid), follow_redirects=True) + res = client.get(url_for("ui.form_clone", uuid=watch_uuid), follow_redirects=True) assert b'Cloned' in res.data # 2 times plus the top link to tag assert res.data.count(b'test-tag') == 3 assert res.data.count(b'another-tag') == 3 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data res = client.get(url_for("tags.delete_all"), follow_redirects=True) @@ -476,5 +476,5 @@ def test_order_of_filters_tag_filter_and_watch_filter(client, live_server, measu """ n += t_index + len(test) - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_ignore_text.py b/changedetectionio/tests/test_ignore_text.py index 298c7df24dd..f9a89631d03 100644 --- a/changedetectionio/tests/test_ignore_text.py +++ b/changedetectionio/tests/test_ignore_text.py @@ -121,7 +121,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa assert bytes(ignore_text.encode('utf-8')) in res.data # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -135,7 +135,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa set_modified_ignore_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -148,7 +148,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa # Just to be sure.. set a regular modified change.. set_modified_original_ignore_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) @@ -161,7 +161,7 @@ def test_check_ignore_text_functionality(client, live_server, measure_memory_usa # it is only ignored, it is not removed (it will be highlighted too) assert b'new ignore stuff' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # When adding some ignore text, it should not trigger a change, even if something else on that line changes @@ -211,7 +211,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem assert bytes(ignore_text.encode('utf-8')) in res.data # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # It should report nothing found (no new 'unviewed' class), adding random ignore text should not cause a change res = client.get(url_for("index")) @@ -224,7 +224,7 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem set_modified_ignore_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -236,10 +236,10 @@ def test_check_global_ignore_text_functionality(client, live_server, measure_mem # Just to be sure.. set a regular modified change that will trigger it set_modified_original_ignore_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_ignorehyperlinks.py b/changedetectionio/tests/test_ignorehyperlinks.py index fba5bd1ac2e..6262121c2b6 100644 --- a/changedetectionio/tests/test_ignorehyperlinks.py +++ b/changedetectionio/tests/test_ignorehyperlinks.py @@ -72,14 +72,14 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag time.sleep(sleep_time_for_fetch_thread) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # set a new html text with a modified link set_modified_ignore_response() time.sleep(sleep_time_for_fetch_thread) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) @@ -101,7 +101,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag assert b"Settings updated." in res.data # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) @@ -119,7 +119,7 @@ def test_render_anchor_tag_content_true(client, live_server, measure_memory_usag assert b"/test-endpoint" in res.data # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_ignorestatuscode.py b/changedetectionio/tests/test_ignorestatuscode.py index a5b2da013fb..1785d3b5015 100644 --- a/changedetectionio/tests/test_ignorestatuscode.py +++ b/changedetectionio/tests/test_ignorestatuscode.py @@ -73,7 +73,7 @@ def test_normal_page_check_works_with_ignore_status_code(client, live_server, me set_some_changed_response() wait_for_all_checks(client) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -121,7 +121,7 @@ def test_403_page_check_works_with_ignore_status_code(client, live_server, measu set_some_changed_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_ignorewhitespace.py b/changedetectionio/tests/test_ignorewhitespace.py index 688b8139d19..a480ee08ddd 100644 --- a/changedetectionio/tests/test_ignorewhitespace.py +++ b/changedetectionio/tests/test_ignorewhitespace.py @@ -80,12 +80,12 @@ def test_check_ignore_whitespace(client, live_server, measure_memory_usage): time.sleep(sleep_time_for_fetch_thread) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) set_original_ignore_response_but_with_whitespace() time.sleep(sleep_time_for_fetch_thread) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up time.sleep(sleep_time_for_fetch_thread) diff --git a/changedetectionio/tests/test_import.py b/changedetectionio/tests/test_import.py index 4b25d65426c..a383b5c9546 100644 --- a/changedetectionio/tests/test_import.py +++ b/changedetectionio/tests/test_import.py @@ -28,7 +28,7 @@ def test_import(client, live_server, measure_memory_usage): assert b"3 Imported" in res.data assert b"tag1" in res.data assert b"other tag" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) # Clear flask alerts res = client.get( url_for("index")) @@ -53,7 +53,7 @@ def xtest_import_skip_url(client, live_server, measure_memory_usage): assert b"1 Imported" in res.data assert b"ht000000broken" in res.data assert b"1 Skipped" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) # Clear flask alerts res = client.get( url_for("index")) @@ -82,7 +82,7 @@ def test_import_distillio(client, live_server, measure_memory_usage): # Give the endpoint time to spin up time.sleep(1) - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) res = client.post( url_for("import_page"), data={ @@ -119,7 +119,7 @@ def test_import_distillio(client, live_server, measure_memory_usage): assert b"nice stuff" in res.data assert b"nerd-news" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) # Clear flask alerts res = client.get(url_for("index")) @@ -169,7 +169,7 @@ def test_import_custom_xlsx(client, live_server, measure_memory_usage): assert filters[0] == '/html[1]/body[1]/div[4]/div[1]/div[1]/div[1]||//*[@id=\'content\']/div[3]/div[1]/div[1]||//*[@id=\'content\']/div[1]' assert watch.get('time_between_check') == {'weeks': 0, 'days': 1, 'hours': 6, 'minutes': 24, 'seconds': 0} - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_import_watchete_xlsx(client, live_server, measure_memory_usage): @@ -214,5 +214,5 @@ def test_import_watchete_xlsx(client, live_server, measure_memory_usage): if watch.get('title') == 'system default website': assert watch.get('fetch_backend') == 'system' # uses default if blank - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_jsonpath_jq_selector.py b/changedetectionio/tests/test_jsonpath_jq_selector.py index e6190e1e7f2..4c98fcfc5e7 100644 --- a/changedetectionio/tests/test_jsonpath_jq_selector.py +++ b/changedetectionio/tests/test_jsonpath_jq_selector.py @@ -229,7 +229,7 @@ def test_check_json_without_filter(client, live_server, measure_memory_usage): assert b'"html": "<b>"' in res.data assert res.data.count(b'{') >= 2 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def check_json_filter(json_filter, client, live_server): @@ -276,7 +276,7 @@ def check_json_filter(json_filter, client, live_server): set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -291,7 +291,7 @@ def check_json_filter(json_filter, client, live_server): # And #462 - check we see the proper utf-8 string there assert "Örnsköldsvik".encode('utf-8') in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_check_jsonpath_filter(client, live_server, measure_memory_usage): @@ -341,7 +341,7 @@ def check_json_filter_bool_val(json_filter, client, live_server): set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -349,7 +349,7 @@ def check_json_filter_bool_val(json_filter, client, live_server): # But the change should be there, tho its hard to test the change was detected because it will show old and new versions assert b'false' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_check_jsonpath_filter_bool_val(client, live_server, measure_memory_usage): @@ -412,7 +412,7 @@ def check_json_ext_filter(json_filter, client, live_server): set_modified_ext_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -427,7 +427,7 @@ def check_json_ext_filter(json_filter, client, live_server): assert b'ForSale' not in res.data assert b'Sold' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_ignore_json_order(client, live_server, measure_memory_usage): @@ -452,7 +452,7 @@ def test_ignore_json_order(client, live_server, measure_memory_usage): f.write('{"world" : 123, "hello": 123}') # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) @@ -463,13 +463,13 @@ def test_ignore_json_order(client, live_server, measure_memory_usage): f.write('{"world" : 123, "hello": 124}') # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_correct_header_detect(client, live_server, measure_memory_usage): @@ -501,7 +501,7 @@ def test_correct_header_detect(client, live_server, measure_memory_usage): assert b'"hello": 123,' in res.data assert b'"world": 123' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_check_jsonpath_ext_filter(client, live_server, measure_memory_usage): diff --git a/changedetectionio/tests/test_live_preview.py b/changedetectionio/tests/test_live_preview.py index 3a4ab989e99..2bd7949feaa 100644 --- a/changedetectionio/tests/test_live_preview.py +++ b/changedetectionio/tests/test_live_preview.py @@ -74,5 +74,5 @@ def test_content_filter_live_preview(client, live_server, measure_memory_usage): assert reply.get('ignore_line_numbers') == [2] # Ignored - "socks" on line 2 assert reply.get('trigger_line_numbers') == [1] # Triggers "Awesome" in line 1 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_nonrenderable_pages.py b/changedetectionio/tests/test_nonrenderable_pages.py index 317667ca478..9a471918448 100644 --- a/changedetectionio/tests/test_nonrenderable_pages.py +++ b/changedetectionio/tests/test_nonrenderable_pages.py @@ -57,7 +57,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # this should not trigger a change, because no good text could be converted from the HTML set_nonrenderable_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -87,7 +87,7 @@ def test_check_basic_change_detection_functionality(client, live_server, measure set_modified_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -95,11 +95,11 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # It should report nothing found (no new 'unviewed' class) res = client.get(url_for("index")) assert b'unviewed' in res.data - client.get(url_for("mark_all_viewed"), follow_redirects=True) + client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) # A totally zero byte (#2528) response should also not trigger an error set_zero_byte_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # 2877 assert watch.last_changed == watch['last_checked'] @@ -111,6 +111,6 @@ def test_check_basic_change_detection_functionality(client, live_server, measure # # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_notification.py b/changedetectionio/tests/test_notification.py index 76e5ce9190d..dd695749933 100644 --- a/changedetectionio/tests/test_notification.py +++ b/changedetectionio/tests/test_notification.py @@ -134,7 +134,7 @@ def test_check_notification(client, live_server, measure_memory_usage): set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(3) @@ -188,7 +188,7 @@ def test_check_notification(client, live_server, measure_memory_usage): # This should insert the {current_snapshot} set_more_modified_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(3) # Verify what was sent as a notification, this file should exist with open("test-datastore/notification.txt", "r") as f: @@ -201,11 +201,11 @@ def test_check_notification(client, live_server, measure_memory_usage): os.unlink("test-datastore/notification.txt") # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) assert os.path.exists("test-datastore/notification.txt") == False @@ -239,7 +239,7 @@ def test_check_notification(client, live_server, measure_memory_usage): # cleanup for the next client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) @@ -276,7 +276,7 @@ def test_notification_validation(client, live_server, measure_memory_usage): # cleanup for the next client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) @@ -321,7 +321,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me wait_for_all_checks(client) set_modified_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) time.sleep(2) # plus extra delay for notifications to fire @@ -361,7 +361,7 @@ def test_notification_custom_endpoint_and_jinja2(client, live_server, measure_me os.unlink("test-datastore/notification-url.txt") client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) @@ -439,7 +439,7 @@ def test_global_send_test_notification(client, live_server, measure_memory_usage assert 'change detection is cool 网站监测 内容更新了' in x client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) @@ -496,8 +496,8 @@ def _test_color_notifications(client, notification_body_token): set_modified_response() - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) time.sleep(3) @@ -508,7 +508,7 @@ def _test_color_notifications(client, notification_body_token): client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) diff --git a/changedetectionio/tests/test_notification_errors.py b/changedetectionio/tests/test_notification_errors.py index 292ef1c6616..8ed1bdc5f35 100644 --- a/changedetectionio/tests/test_notification_errors.py +++ b/changedetectionio/tests/test_notification_errors.py @@ -69,4 +69,4 @@ def test_check_notification_error_handling(client, live_server, measure_memory_u os.unlink("test-datastore/notification.txt") assert 'xxxxx' in notification_submission - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) diff --git a/changedetectionio/tests/test_pdf.py b/changedetectionio/tests/test_pdf.py index 1f7be0ff4ed..c340e2fca2c 100644 --- a/changedetectionio/tests/test_pdf.py +++ b/changedetectionio/tests/test_pdf.py @@ -44,8 +44,8 @@ def test_fetch_pdf(client, live_server, measure_memory_usage): shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf") changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_preview_endpoints.py b/changedetectionio/tests/test_preview_endpoints.py index e1c8c747f83..01ab6ddef00 100644 --- a/changedetectionio/tests/test_preview_endpoints.py +++ b/changedetectionio/tests/test_preview_endpoints.py @@ -42,8 +42,8 @@ def test_fetch_pdf(client, live_server, measure_memory_usage): shutil.copy("tests/test2.pdf", "test-datastore/endpoint-test.pdf") changed_md5 = hashlib.md5(open("test-datastore/endpoint-test.pdf", 'rb').read()).hexdigest().upper() - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) diff --git a/changedetectionio/tests/test_request.py b/changedetectionio/tests/test_request.py index 93714e64750..351b6f331a8 100644 --- a/changedetectionio/tests/test_request.py +++ b/changedetectionio/tests/test_request.py @@ -82,7 +82,7 @@ def test_headers_in_request(client, live_server, measure_memory_usage): for k, watch in client.application.config.get('DATASTORE').data.get('watching').items(): assert 'custom' in watch.get('remote_server_reply') # added in util.py - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_body_in_request(client, live_server, measure_memory_usage): @@ -177,7 +177,7 @@ def test_body_in_request(client, live_server, measure_memory_usage): follow_redirects=True ) assert b"Body must be empty when Request Method is set to GET" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_method_in_request(client, live_server, measure_memory_usage): @@ -253,7 +253,7 @@ def test_method_in_request(client, live_server, measure_memory_usage): # Should be only one with method set to PATCH assert watches_with_method == 1 - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # Re #2408 - user-agent override test, also should handle case-insensitive header deduplication @@ -309,7 +309,7 @@ def test_ua_global_override(client, live_server, measure_memory_usage): ) assert b"agent-from-watch" in res.data assert b"html-requests-user-agent" not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_headers_textfile_in_request(client, live_server, measure_memory_usage): @@ -383,7 +383,7 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): f.write("watch-header: nice\r\nurl-header-watch: http://example.com/watch") wait_for_all_checks(client) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up, this actually is not super reliable and pytest can terminate before the check is ran wait_for_all_checks(client) @@ -422,5 +422,5 @@ def test_headers_textfile_in_request(client, live_server, measure_memory_usage): assert "User-Agent:".encode('utf-8') + requests_ua.encode('utf-8') in res.data # unlink headers.txt on start/stop - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_restock_itemprop.py b/changedetectionio/tests/test_restock_itemprop.py index 9c9a0e88c1e..8beae6fd058 100644 --- a/changedetectionio/tests/test_restock_itemprop.py +++ b/changedetectionio/tests/test_restock_itemprop.py @@ -69,7 +69,7 @@ def test_restock_itemprop_basic(client, live_server): assert b'has-restock-info' in res.data assert b' in-stock' in res.data assert b' not-in-stock' not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -85,7 +85,7 @@ def test_restock_itemprop_basic(client, live_server): assert b'has-restock-info not-in-stock' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_itemprop_price_change(client, live_server): @@ -108,12 +108,12 @@ def test_itemprop_price_change(client, live_server): # basic price change, look for notification set_original_response(props_markup=instock_props[0], price='180.45') - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'180.45' in res.data assert b'unviewed' in res.data - client.get(url_for("mark_all_viewed"), follow_redirects=True) + client.get(url_for("ui.mark_all_viewed"), follow_redirects=True) # turning off price change trigger, but it should show the new price, with no change notification set_original_response(props_markup=instock_props[0], price='120.45') @@ -123,19 +123,19 @@ def test_itemprop_price_change(client, live_server): follow_redirects=True ) assert b"Updated watch." in res.data - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'120.45' in res.data assert b'unviewed' not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def _run_test_minmax_limit(client, extra_watch_edit_form): - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data test_url = url_for('test_endpoint', _external=True) @@ -164,11 +164,11 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): assert b"Updated watch." in res.data wait_for_all_checks(client) - client.get(url_for("mark_all_viewed")) + client.get(url_for("ui.mark_all_viewed")) # price changed to something greater than min (900), BUT less than max (1100).. should be no change set_original_response(props_markup=instock_props[0], price='1000.45') - client.get(url_for("form_watch_checknow")) + client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("index")) @@ -180,36 +180,36 @@ def _run_test_minmax_limit(client, extra_watch_edit_form): # price changed to something LESS than min (900), SHOULD be a change set_original_response(props_markup=instock_props[0], price='890.45') - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) res = client.get(url_for("index")) assert b'890.45' in res.data assert b'unviewed' in res.data - client.get(url_for("mark_all_viewed")) + client.get(url_for("ui.mark_all_viewed")) # 2715 - Price detection (once it crosses the "lower" threshold) again with a lower price - should trigger again! set_original_response(props_markup=instock_props[0], price='820.45') - res = client.get(url_for("form_watch_checknow"), follow_redirects=True) - assert b'1 watches queued for rechecking.' in res.data + res = client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) + assert b'Queued 1 watch for rechecking.' in res.data wait_for_all_checks(client) res = client.get(url_for("index")) assert b'820.45' in res.data assert b'unviewed' in res.data - client.get(url_for("mark_all_viewed")) + client.get(url_for("ui.mark_all_viewed")) # price changed to something MORE than max (1100.10), SHOULD be a change set_original_response(props_markup=instock_props[0], price='1890.45') - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) # Depending on the LOCALE it may be either of these (generally for US/default/etc) assert b'1,890.45' in res.data or b'1890.45' in res.data assert b'unviewed' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -254,7 +254,7 @@ def test_restock_itemprop_with_tag(client, live_server): def test_itemprop_percent_threshold(client, live_server): #live_server_setup(live_server) - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data test_url = url_for('test_endpoint', _external=True) @@ -286,7 +286,7 @@ def test_itemprop_percent_threshold(client, live_server): # Basic change should not trigger set_original_response(props_markup=instock_props[0], price='960.45') - client.get(url_for("form_watch_checknow")) + client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'960.45' in res.data @@ -294,7 +294,7 @@ def test_itemprop_percent_threshold(client, live_server): # Bigger INCREASE change than the threshold should trigger set_original_response(props_markup=instock_props[0], price='1960.45') - client.get(url_for("form_watch_checknow")) + client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'1,960.45' or b'1960.45' in res.data #depending on locale @@ -302,9 +302,9 @@ def test_itemprop_percent_threshold(client, live_server): # Small decrease should NOT trigger - client.get(url_for("mark_all_viewed")) + client.get(url_for("ui.mark_all_viewed")) set_original_response(props_markup=instock_props[0], price='1950.45') - client.get(url_for("form_watch_checknow")) + client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'1,950.45' or b'1950.45' in res.data #depending on locale @@ -313,7 +313,7 @@ def test_itemprop_percent_threshold(client, live_server): - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -369,7 +369,7 @@ def test_change_with_notification_values(client, live_server): set_original_response(props_markup=instock_props[0], price='960.45') # A change in price, should trigger a change by default set_original_response(props_markup=instock_props[0], price='1950.45') - client.get(url_for("form_watch_checknow")) + client.get(url_for("ui.form_watch_checknow")) wait_for_all_checks(client) wait_for_notification_endpoint_output() assert os.path.isfile("test-datastore/notification.txt"), "Notification received" @@ -389,7 +389,7 @@ def test_change_with_notification_values(client, live_server): def test_data_sanity(client, live_server): #live_server_setup(live_server) - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data test_url = url_for('test_endpoint', _external=True) @@ -417,7 +417,7 @@ def test_data_sanity(client, live_server): assert str(res.data.decode()).count("950.95") == 1, "Price should only show once (for the watch added, no other watches yet)" ## different test, check the edit page works on an empty request result - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data client.post( @@ -431,7 +431,7 @@ def test_data_sanity(client, live_server): url_for("edit_page", uuid="first")) assert test_url2.encode('utf-8') in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # All examples should give a prive of 666.66 diff --git a/changedetectionio/tests/test_rss.py b/changedetectionio/tests/test_rss.py index 69ce479992d..9704e11466e 100644 --- a/changedetectionio/tests/test_rss.py +++ b/changedetectionio/tests/test_rss.py @@ -70,7 +70,7 @@ def test_rss_and_token(client, live_server, measure_memory_usage): wait_for_all_checks(client) set_modified_response() time.sleep(1) - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) # Add our URL to the import page @@ -88,7 +88,7 @@ def test_rss_and_token(client, live_server, measure_memory_usage): assert b"Access denied, bad token" not in res.data assert b"Random content" in res.data - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage): #live_server_setup(live_server) @@ -116,7 +116,7 @@ def test_basic_cdata_rss_markup(client, live_server, measure_memory_usage): assert b'\nsomething 123") - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_trigger_regex_with_filter.py b/changedetectionio/tests/test_trigger_regex_with_filter.py index 4e3eba729e2..0ccda27ca40 100644 --- a/changedetectionio/tests/test_trigger_regex_with_filter.py +++ b/changedetectionio/tests/test_trigger_regex_with_filter.py @@ -63,7 +63,7 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me with open("test-datastore/endpoint-content.txt", "w") as f: f.write("some new noise with cool stuff2 ok") - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(sleep_time_for_fetch_thread) # It should report nothing found (nothing should match the regex and filter) @@ -74,11 +74,11 @@ def test_trigger_regex_functionality_with_filter(client, live_server, measure_me with open("test-datastore/endpoint-content.txt", "w") as f: f.write("some new noise with cool stuff6 ok") - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) time.sleep(sleep_time_for_fetch_thread) res = client.get(url_for("index")) assert b'unviewed' in res.data # Cleanup everything - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data diff --git a/changedetectionio/tests/test_unique_lines.py b/changedetectionio/tests/test_unique_lines.py index f4148157210..978e92245ca 100644 --- a/changedetectionio/tests/test_unique_lines.py +++ b/changedetectionio/tests/test_unique_lines.py @@ -102,7 +102,7 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage): set_modified_swapped_lines() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -113,11 +113,11 @@ def test_unique_lines_functionality(client, live_server, measure_memory_usage): # Now set the content which contains the new text and re-ordered existing text set_modified_with_trigger_text_response() - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_sort_lines_functionality(client, live_server, measure_memory_usage): @@ -147,7 +147,7 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage): # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -166,7 +166,7 @@ def test_sort_lines_functionality(client, live_server, measure_memory_usage): assert res.data.find(b'A uppercase') < res.data.find(b'Z last') assert res.data.find(b'Some initial text') < res.data.find(b'Which is across multiple lines') - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -199,7 +199,7 @@ def test_extra_filters(client, live_server, measure_memory_usage): # Give the thread time to pick it up wait_for_all_checks(client) # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) @@ -213,5 +213,5 @@ def test_extra_filters(client, live_server, measure_memory_usage): # still should remain unsorted ('A - sortable line') stays at the end assert res.data.find(b'A - sortable line') > res.data.find(b'Which is across multiple lines') - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data \ No newline at end of file diff --git a/changedetectionio/tests/test_xpath_selector.py b/changedetectionio/tests/test_xpath_selector.py index 4f50ad0d75e..c4d97808f79 100644 --- a/changedetectionio/tests/test_xpath_selector.py +++ b/changedetectionio/tests/test_xpath_selector.py @@ -100,7 +100,7 @@ def test_check_xpath_filter_utf8(client, live_server, measure_memory_usage): wait_for_all_checks(client) res = client.get(url_for("index")) assert b'Unicode strings with encoding declaration are not supported.' not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -164,7 +164,7 @@ def test_check_xpath_text_function_utf8(client, live_server, measure_memory_usag assert b'Stock Alert (UK): RPi CM4' in res.data assert b'Stock Alert (UK): Big monitor' in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -204,13 +204,13 @@ def test_check_markup_xpath_filter_restriction(client, live_server, measure_memo set_modified_response() # Trigger a check - client.get(url_for("form_watch_checknow"), follow_redirects=True) + client.get(url_for("ui.form_watch_checknow"), follow_redirects=True) # Give the thread time to pick it up wait_for_all_checks(client) res = client.get(url_for("index")) assert b'unviewed' not in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -231,7 +231,7 @@ def test_xpath_validation(client, live_server, measure_memory_usage): follow_redirects=True ) assert b"is not a valid XPath expression" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -252,7 +252,7 @@ def test_xpath23_prefix_validation(client, live_server, measure_memory_usage): follow_redirects=True ) assert b"is not a valid XPath expression" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data def test_xpath1_lxml(client, live_server, measure_memory_usage): @@ -336,13 +336,13 @@ def test_xpath1_validation(client, live_server, measure_memory_usage): follow_redirects=True ) assert b"is not a valid XPath expression" in res.data - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data # actually only really used by the distll.io importer, but could be handy too def test_check_with_prefix_include_filters(client, live_server, measure_memory_usage): - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data set_original_response() @@ -375,7 +375,7 @@ def test_check_with_prefix_include_filters(client, live_server, measure_memory_u assert b"Some text thats the same" in res.data # in selector assert b"Some text that will change" not in res.data # not in selector - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_various_rules(client, live_server, measure_memory_usage): @@ -422,7 +422,7 @@ def test_various_rules(client, live_server, measure_memory_usage): res = client.get(url_for("index")) assert b'fetch-error' not in res.data, f"Should not see errors after '{r} filter" - res = client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + res = client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) assert b'Deleted' in res.data @@ -460,7 +460,7 @@ def test_xpath_20(client, live_server, measure_memory_usage): assert b"Some text thats the same" in res.data # in selector assert b"Some text that will change" in res.data # in selector - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_xpath_20_function_count(client, live_server, measure_memory_usage): @@ -496,7 +496,7 @@ def test_xpath_20_function_count(client, live_server, measure_memory_usage): assert b"246913579975308642" in res.data # in selector - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_xpath_20_function_count2(client, live_server, measure_memory_usage): @@ -532,7 +532,7 @@ def test_xpath_20_function_count2(client, live_server, measure_memory_usage): assert b"246913579975308642" in res.data # in selector - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) def test_xpath_20_function_string_join_matches(client, live_server, measure_memory_usage): @@ -569,5 +569,5 @@ def test_xpath_20_function_string_join_matches(client, live_server, measure_memo assert b"Some text thats the samespecialconjunctionSome text that will change" in res.data # in selector - client.get(url_for("form_delete", uuid="all"), follow_redirects=True) + client.get(url_for("ui.form_delete", uuid="all"), follow_redirects=True) diff --git a/changedetectionio/tests/visualselector/test_fetch_data.py b/changedetectionio/tests/visualselector/test_fetch_data.py index cf2451ae643..60788bab6b2 100644 --- a/changedetectionio/tests/visualselector/test_fetch_data.py +++ b/changedetectionio/tests/visualselector/test_fetch_data.py @@ -83,7 +83,7 @@ def test_visual_selector_content_ready(client, live_server, measure_memory_usage ) assert b'notification_screenshot' in res.data client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) @@ -188,6 +188,6 @@ def test_non_200_errors_report_browsersteps(client, live_server): assert b'Error - 404' in res.data client.get( - url_for("form_delete", uuid="all"), + url_for("ui.form_delete", uuid="all"), follow_redirects=True ) From 896fff7f2b2e23022b9ed9545c3af7c6f0994552 Mon Sep 17 00:00:00 2001 From: dgtlmoon Date: Tue, 18 Mar 2025 01:04:16 +0100 Subject: [PATCH 07/18] Moving edit UI to its own blueprint --- changedetectionio/blueprint/ui/__init__.py | 5 + changedetectionio/blueprint/ui/edit.py | 333 ++++++++++++++++++ changedetectionio/flask_app.py | 317 ----------------- changedetectionio/model/Watch.py | 2 +- changedetectionio/templates/base.html | 2 +- changedetectionio/templates/diff.html | 2 +- changedetectionio/templates/edit.html | 6 +- changedetectionio/templates/preview.html | 2 +- .../templates/watch-overview.html | 2 +- .../test_custom_browser_url.py | 4 +- .../fetchers/test_custom_js_before_content.py | 2 +- .../tests/proxy_list/test_multiple_proxy.py | 2 +- .../tests/proxy_list/test_noproxy.py | 6 +- .../tests/proxy_socks5/test_socks5_proxy.py | 4 +- .../proxy_socks5/test_socks5_proxy_sources.py | 4 +- .../tests/smtp/test_notification_smtp.py | 2 +- .../tests/test_add_replace_remove_filter.py | 4 +- changedetectionio/tests/test_api.py | 4 +- changedetectionio/tests/test_auth.py | 2 +- changedetectionio/tests/test_backend.py | 2 +- .../tests/test_block_while_text_present.py | 4 +- changedetectionio/tests/test_conditions.py | 2 +- changedetectionio/tests/test_css_selector.py | 8 +- .../tests/test_element_removal.py | 6 +- changedetectionio/tests/test_errorhandling.py | 2 +- changedetectionio/tests/test_extract_regex.py | 6 +- .../tests/test_filter_exist_changes.py | 2 +- .../tests/test_filter_failure_notification.py | 4 +- changedetectionio/tests/test_group.py | 4 +- changedetectionio/tests/test_ignore.py | 4 +- changedetectionio/tests/test_ignore_text.py | 6 +- .../tests/test_ignorestatuscode.py | 2 +- changedetectionio/tests/test_import.py | 2 +- .../tests/test_jsonpath_jq_selector.py | 10 +- changedetectionio/tests/test_live_preview.py | 6 +- changedetectionio/tests/test_notification.py | 8 +- .../tests/test_notification_errors.py | 2 +- changedetectionio/tests/test_request.py | 18 +- .../tests/test_restock_itemprop.py | 8 +- changedetectionio/tests/test_rss.py | 2 +- changedetectionio/tests/test_scheduler.py | 8 +- changedetectionio/tests/test_search.py | 4 +- changedetectionio/tests/test_security.py | 2 +- changedetectionio/tests/test_share_watch.py | 8 +- changedetectionio/tests/test_source.py | 2 +- changedetectionio/tests/test_trigger.py | 4 +- changedetectionio/tests/test_trigger_regex.py | 2 +- .../tests/test_trigger_regex_with_filter.py | 2 +- changedetectionio/tests/test_unique_lines.py | 6 +- .../tests/test_watch_fields_storage.py | 4 +- .../tests/test_xpath_selector.py | 26 +- .../tests/visualselector/test_fetch_data.py | 8 +- 52 files changed, 455 insertions(+), 434 deletions(-) create mode 100644 changedetectionio/blueprint/ui/edit.py diff --git a/changedetectionio/blueprint/ui/__init__.py b/changedetectionio/blueprint/ui/__init__.py index 19790ac78ac..45d11f50524 100644 --- a/changedetectionio/blueprint/ui/__init__.py +++ b/changedetectionio/blueprint/ui/__init__.py @@ -4,10 +4,15 @@ from functools import wraps from changedetectionio.store import ChangeDetectionStore +from changedetectionio.blueprint.ui.edit import construct_blueprint as construct_edit_blueprint def construct_blueprint(datastore: ChangeDetectionStore, update_q, running_update_threads, queuedWatchMetaData): ui_blueprint = Blueprint('ui', __name__, template_folder="templates") + # Register the edit blueprint + edit_blueprint = construct_edit_blueprint(datastore, update_q, queuedWatchMetaData) + ui_blueprint.register_blueprint(edit_blueprint) + # Import the login decorator from changedetectionio.auth_decorator import login_optionally_required diff --git a/changedetectionio/blueprint/ui/edit.py b/changedetectionio/blueprint/ui/edit.py new file mode 100644 index 00000000000..e02b5f16c31 --- /dev/null +++ b/changedetectionio/blueprint/ui/edit.py @@ -0,0 +1,333 @@ +import time +from copy import deepcopy +import os +import importlib.resources +from flask import Blueprint, request, redirect, url_for, flash, render_template, make_response, send_from_directory +from loguru import logger +from jinja2 import Environment, FileSystemLoader + +from changedetectionio.store import ChangeDetectionStore +from changedetectionio.auth_decorator import login_optionally_required +from changedetectionio.time_handler import is_within_schedule + +def construct_blueprint(datastore: ChangeDetectionStore, update_q, queuedWatchMetaData): + edit_blueprint = Blueprint('ui_edit', __name__, template_folder="../ui/templates") + + def _watch_has_tag_options_set(watch): + """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" + for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): + if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')): + return True + + @edit_blueprint.route("/edit/", methods=['GET', 'POST']) + @login_optionally_required + # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists + # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? + def edit_page(uuid): + from changedetectionio import forms + from changedetectionio.blueprint.browser_steps.browser_steps import browser_step_ui_config + from changedetectionio import processors + import importlib + + # More for testing, possible to return the first/only + if not datastore.data['watching'].keys(): + flash("No watches to edit", "error") + return redirect(url_for('index')) + + if uuid == 'first': + uuid = list(datastore.data['watching'].keys()).pop() + + if not uuid in datastore.data['watching']: + flash("No watch with the UUID %s found." % (uuid), "error") + return redirect(url_for('index')) + + switch_processor = request.args.get('switch_processor') + if switch_processor: + for p in processors.available_processors(): + if p[0] == switch_processor: + datastore.data['watching'][uuid]['processor'] = switch_processor + flash(f"Switched to mode - {p[1]}.") + datastore.clear_watch_history(uuid) + redirect(url_for('ui_edit.edit_page', uuid=uuid)) + + # be sure we update with a copy instead of accidently editing the live object by reference + default = deepcopy(datastore.data['watching'][uuid]) + + # Defaults for proxy choice + if datastore.proxy_list is not None: # When enabled + # @todo + # Radio needs '' not None, or incase that the chosen one no longer exists + if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list): + default['proxy'] = '' + # proxy_override set to the json/text list of the items + + # Does it use some custom form? does one exist? + processor_name = datastore.data['watching'][uuid].get('processor', '') + processor_classes = next((tpl for tpl in processors.find_processors() if tpl[1] == processor_name), None) + if not processor_classes: + flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error') + return redirect(url_for('index')) + + parent_module = processors.get_parent_module(processor_classes[0]) + + try: + # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code) + forms_module = importlib.import_module(f"{parent_module.__name__}.forms") + # Access the 'processor_settings_form' class from the 'forms' module + form_class = getattr(forms_module, 'processor_settings_form') + except ModuleNotFoundError as e: + # .forms didnt exist + form_class = forms.processor_text_json_diff_form + except AttributeError as e: + # .forms exists but no useful form + form_class = forms.processor_text_json_diff_form + + form = form_class(formdata=request.form if request.method == 'POST' else None, + data=default, + extra_notification_tokens=default.extra_notification_token_values(), + default_system_settings=datastore.data['settings'] + ) + + # For the form widget tag UUID back to "string name" for the field + form.tags.datastore = datastore + + # Used by some forms that need to dig deeper + form.datastore = datastore + form.watch = default + + for p in datastore.extra_browsers: + form.fetch_backend.choices.append(p) + + form.fetch_backend.choices.append(("system", 'System settings default')) + + # form.browser_steps[0] can be assumed that we 'goto url' first + + if datastore.proxy_list is None: + # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead + del form.proxy + else: + form.proxy.choices = [('', 'Default')] + for p in datastore.proxy_list: + form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) + + + if request.method == 'POST' and form.validate(): + + # If they changed processor, it makes sense to reset it. + if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'): + datastore.data['watching'][uuid].clear_watch() + flash("Reset watch history due to change of processor") + + extra_update_obj = { + 'consecutive_filter_failures': 0, + 'last_error' : False + } + + if request.args.get('unpause_on_save'): + extra_update_obj['paused'] = False + + extra_update_obj['time_between_check'] = form.time_between_check.data + + # Ignore text + form_ignore_text = form.ignore_text.data + datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text + + # Be sure proxy value is None + if datastore.proxy_list is not None and form.data['proxy'] == '': + extra_update_obj['proxy'] = None + + # Unsetting all filter_text methods should make it go back to default + # This particularly affects tests running + if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \ + and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \ + and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'): + extra_update_obj['filter_text_added'] = True + extra_update_obj['filter_text_replaced'] = True + extra_update_obj['filter_text_removed'] = True + + # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs + tag_uuids = [] + if form.data.get('tags'): + # Sometimes in testing this can be list, dont know why + if type(form.data.get('tags')) == list: + extra_update_obj['tags'] = form.data.get('tags') + else: + for t in form.data.get('tags').split(','): + tag_uuids.append(datastore.add_tag(name=t)) + extra_update_obj['tags'] = tag_uuids + + datastore.data['watching'][uuid].update(form.data) + datastore.data['watching'][uuid].update(extra_update_obj) + + if not datastore.data['watching'][uuid].get('tags'): + # Force it to be a list, because form.data['tags'] will be string if nothing found + # And del(form.data['tags'] ) wont work either for some reason + datastore.data['watching'][uuid]['tags'] = [] + + # Recast it if need be to right data Watch handler + watch_class = processors.get_custom_watch_obj_for_processor(form.data.get('processor')) + datastore.data['watching'][uuid] = watch_class(datastore_path=datastore.datastore_path, default=datastore.data['watching'][uuid]) + flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") + + # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds + # But in the case something is added we should save straight away + datastore.needs_write_urgent = True + + # Do not queue on edit if its not within the time range + + # @todo maybe it should never queue anyway on edit... + is_in_schedule = True + watch = datastore.data['watching'].get(uuid) + + if watch.get('time_between_check_use_default'): + time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {}) + else: + time_schedule_limit = watch.get('time_schedule_limit') + + tz_name = time_schedule_limit.get('timezone') + if not tz_name: + tz_name = datastore.data['settings']['application'].get('timezone', 'UTC') + + if time_schedule_limit and time_schedule_limit.get('enabled'): + try: + is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit, + default_tz=tz_name + ) + except Exception as e: + logger.error( + f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}") + return False + + ############################# + if not datastore.data['watching'][uuid].get('paused') and is_in_schedule: + # Queue the watch for immediate recheck, with a higher priority + update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) + + # Diff page [edit] link should go back to diff page + if request.args.get("next") and request.args.get("next") == 'diff': + return redirect(url_for('diff_history_page', uuid=uuid)) + + return redirect(url_for('index', tag=request.args.get("tag",''))) + + else: + if request.method == 'POST' and not form.validate(): + flash("An error occurred, please see below.", "error") + + visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid) + + + # JQ is difficult to install on windows and must be manually added (outside requirements.txt) + jq_support = True + try: + import jq + except ModuleNotFoundError: + jq_support = False + + watch = datastore.data['watching'].get(uuid) + + system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' + + watch_uses_webdriver = False + if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): + watch_uses_webdriver = True + + from zoneinfo import available_timezones + + # Only works reliably with Playwright + + template_args = { + 'available_processors': processors.available_processors(), + 'available_timezones': sorted(available_timezones()), + 'browser_steps_config': browser_step_ui_config, + 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), + 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), + 'extra_processor_config': form.extra_tab_content(), + 'extra_title': f" - Edit - {watch.label}", + 'form': form, + 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, + 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, + 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), + 'watch_uses_webdriver': watch_uses_webdriver, + 'jq_support': jq_support, + 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), + 'settings_application': datastore.data['settings']['application'], + 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), + 'using_global_webdriver_wait': not default['webdriver_delay'], + 'uuid': uuid, + 'watch': watch + } + + included_content = None + if form.extra_form_content(): + # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/ + # And then render the code from the module + templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates')) + env = Environment(loader=FileSystemLoader(templates_dir)) + template = env.from_string(form.extra_form_content()) + included_content = template.render(**template_args) + + output = render_template("edit.html", + extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, + extra_form_content=included_content, + **template_args + ) + + return output + + @edit_blueprint.route("/edit//get-html", methods=['GET']) + @login_optionally_required + def watch_get_latest_html(uuid): + from io import BytesIO + from flask import send_file + import brotli + + watch = datastore.data['watching'].get(uuid) + if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir): + latest_filename = list(watch.history.keys())[-1] + html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br") + with open(html_fname, 'rb') as f: + if html_fname.endswith('.br'): + # Read and decompress the Brotli file + decompressed_data = brotli.decompress(f.read()) + else: + decompressed_data = f.read() + + buffer = BytesIO(decompressed_data) + + return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html') + + # Return a 500 error + abort(500) + + # Ajax callback + @edit_blueprint.route("/edit//preview-rendered", methods=['POST']) + @login_optionally_required + def watch_get_preview_rendered(uuid): + '''For when viewing the "preview" of the rendered text from inside of Edit''' + from flask import jsonify + from changedetectionio.processors.text_json_diff import prepare_filter_prevew + result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) + return jsonify(result) + + @edit_blueprint.route("/highlight_submit_ignore_url", methods=['POST']) + @login_optionally_required + def highlight_submit_ignore_url(): + import re + mode = request.form.get('mode') + selection = request.form.get('selection') + + uuid = request.args.get('uuid','') + if datastore.data["watching"].get(uuid): + if mode == 'exact': + for l in selection.splitlines(): + datastore.data["watching"][uuid]['ignore_text'].append(l.strip()) + elif mode == 'digit-regex': + for l in selection.splitlines(): + # Replace any series of numbers with a regex + s = re.escape(l.strip()) + s = re.sub(r'[0-9]+', r'\\d+', s) + datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/') + + return f"Click to preview" + + return edit_blueprint \ No newline at end of file diff --git a/changedetectionio/flask_app.py b/changedetectionio/flask_app.py index 023aa35d377..8928ef0f5df 100644 --- a/changedetectionio/flask_app.py +++ b/changedetectionio/flask_app.py @@ -548,268 +548,6 @@ def ajax_callback_send_notification_test(watch_uuid=None): - def _watch_has_tag_options_set(watch): - """This should be fixed better so that Tag is some proper Model, a tag is just a Watch also""" - for tag_uuid, tag in datastore.data['settings']['application'].get('tags', {}).items(): - if tag_uuid in watch.get('tags', []) and (tag.get('include_filters') or tag.get('subtractive_selectors')): - return True - - @app.route("/edit/", methods=['GET', 'POST']) - @login_optionally_required - # https://stackoverflow.com/questions/42984453/wtforms-populate-form-with-data-if-data-exists - # https://wtforms.readthedocs.io/en/3.0.x/forms/#wtforms.form.Form.populate_obj ? - def edit_page(uuid): - from . import forms - from .blueprint.browser_steps.browser_steps import browser_step_ui_config - from . import processors - import importlib - - # More for testing, possible to return the first/only - if not datastore.data['watching'].keys(): - flash("No watches to edit", "error") - return redirect(url_for('index')) - - if uuid == 'first': - uuid = list(datastore.data['watching'].keys()).pop() - - if not uuid in datastore.data['watching']: - flash("No watch with the UUID %s found." % (uuid), "error") - return redirect(url_for('index')) - - switch_processor = request.args.get('switch_processor') - if switch_processor: - for p in processors.available_processors(): - if p[0] == switch_processor: - datastore.data['watching'][uuid]['processor'] = switch_processor - flash(f"Switched to mode - {p[1]}.") - datastore.clear_watch_history(uuid) - redirect(url_for('edit_page', uuid=uuid)) - - # be sure we update with a copy instead of accidently editing the live object by reference - default = deepcopy(datastore.data['watching'][uuid]) - - # Defaults for proxy choice - if datastore.proxy_list is not None: # When enabled - # @todo - # Radio needs '' not None, or incase that the chosen one no longer exists - if default['proxy'] is None or not any(default['proxy'] in tup for tup in datastore.proxy_list): - default['proxy'] = '' - # proxy_override set to the json/text list of the items - - # Does it use some custom form? does one exist? - processor_name = datastore.data['watching'][uuid].get('processor', '') - processor_classes = next((tpl for tpl in find_processors() if tpl[1] == processor_name), None) - if not processor_classes: - flash(f"Cannot load the edit form for processor/plugin '{processor_classes[1]}', plugin missing?", 'error') - return redirect(url_for('index')) - - parent_module = get_parent_module(processor_classes[0]) - - try: - # Get the parent of the "processor.py" go up one, get the form (kinda spaghetti but its reusing existing code) - forms_module = importlib.import_module(f"{parent_module.__name__}.forms") - # Access the 'processor_settings_form' class from the 'forms' module - form_class = getattr(forms_module, 'processor_settings_form') - except ModuleNotFoundError as e: - # .forms didnt exist - form_class = forms.processor_text_json_diff_form - except AttributeError as e: - # .forms exists but no useful form - form_class = forms.processor_text_json_diff_form - - form = form_class(formdata=request.form if request.method == 'POST' else None, - data=default, - extra_notification_tokens=default.extra_notification_token_values(), - default_system_settings=datastore.data['settings'] - ) - - # For the form widget tag UUID back to "string name" for the field - form.tags.datastore = datastore - - # Used by some forms that need to dig deeper - form.datastore = datastore - form.watch = default - - for p in datastore.extra_browsers: - form.fetch_backend.choices.append(p) - - form.fetch_backend.choices.append(("system", 'System settings default')) - - # form.browser_steps[0] can be assumed that we 'goto url' first - - if datastore.proxy_list is None: - # @todo - Couldn't get setattr() etc dynamic addition working, so remove it instead - del form.proxy - else: - form.proxy.choices = [('', 'Default')] - for p in datastore.proxy_list: - form.proxy.choices.append(tuple((p, datastore.proxy_list[p]['label']))) - - - if request.method == 'POST' and form.validate(): - - # If they changed processor, it makes sense to reset it. - if datastore.data['watching'][uuid].get('processor') != form.data.get('processor'): - datastore.data['watching'][uuid].clear_watch() - flash("Reset watch history due to change of processor") - - extra_update_obj = { - 'consecutive_filter_failures': 0, - 'last_error' : False - } - - if request.args.get('unpause_on_save'): - extra_update_obj['paused'] = False - - extra_update_obj['time_between_check'] = form.time_between_check.data - - # Ignore text - form_ignore_text = form.ignore_text.data - datastore.data['watching'][uuid]['ignore_text'] = form_ignore_text - - # Be sure proxy value is None - if datastore.proxy_list is not None and form.data['proxy'] == '': - extra_update_obj['proxy'] = None - - # Unsetting all filter_text methods should make it go back to default - # This particularly affects tests running - if 'filter_text_added' in form.data and not form.data.get('filter_text_added') \ - and 'filter_text_replaced' in form.data and not form.data.get('filter_text_replaced') \ - and 'filter_text_removed' in form.data and not form.data.get('filter_text_removed'): - extra_update_obj['filter_text_added'] = True - extra_update_obj['filter_text_replaced'] = True - extra_update_obj['filter_text_removed'] = True - - # Because wtforms doesn't support accessing other data in process_ , but we convert the CSV list of tags back to a list of UUIDs - tag_uuids = [] - if form.data.get('tags'): - # Sometimes in testing this can be list, dont know why - if type(form.data.get('tags')) == list: - extra_update_obj['tags'] = form.data.get('tags') - else: - for t in form.data.get('tags').split(','): - tag_uuids.append(datastore.add_tag(name=t)) - extra_update_obj['tags'] = tag_uuids - - datastore.data['watching'][uuid].update(form.data) - datastore.data['watching'][uuid].update(extra_update_obj) - - if not datastore.data['watching'][uuid].get('tags'): - # Force it to be a list, because form.data['tags'] will be string if nothing found - # And del(form.data['tags'] ) wont work either for some reason - datastore.data['watching'][uuid]['tags'] = [] - - # Recast it if need be to right data Watch handler - watch_class = get_custom_watch_obj_for_processor(form.data.get('processor')) - datastore.data['watching'][uuid] = watch_class(datastore_path=datastore_o.datastore_path, default=datastore.data['watching'][uuid]) - flash("Updated watch - unpaused!" if request.args.get('unpause_on_save') else "Updated watch.") - - # Re #286 - We wait for syncing new data to disk in another thread every 60 seconds - # But in the case something is added we should save straight away - datastore.needs_write_urgent = True - - # Do not queue on edit if its not within the time range - - # @todo maybe it should never queue anyway on edit... - is_in_schedule = True - watch = datastore.data['watching'].get(uuid) - - if watch.get('time_between_check_use_default'): - time_schedule_limit = datastore.data['settings']['requests'].get('time_schedule_limit', {}) - else: - time_schedule_limit = watch.get('time_schedule_limit') - - tz_name = time_schedule_limit.get('timezone') - if not tz_name: - tz_name = datastore.data['settings']['application'].get('timezone', 'UTC') - - if time_schedule_limit and time_schedule_limit.get('enabled'): - try: - is_in_schedule = is_within_schedule(time_schedule_limit=time_schedule_limit, - default_tz=tz_name - ) - except Exception as e: - logger.error( - f"{uuid} - Recheck scheduler, error handling timezone, check skipped - TZ name '{tz_name}' - {str(e)}") - return False - - ############################# - if not datastore.data['watching'][uuid].get('paused') and is_in_schedule: - # Queue the watch for immediate recheck, with a higher priority - update_q.put(queuedWatchMetaData.PrioritizedItem(priority=1, item={'uuid': uuid})) - - # Diff page [edit] link should go back to diff page - if request.args.get("next") and request.args.get("next") == 'diff': - return redirect(url_for('diff_history_page', uuid=uuid)) - - return redirect(url_for('index', tag=request.args.get("tag",''))) - - else: - if request.method == 'POST' and not form.validate(): - flash("An error occurred, please see below.", "error") - - visualselector_data_is_ready = datastore.visualselector_data_is_ready(uuid) - - - # JQ is difficult to install on windows and must be manually added (outside requirements.txt) - jq_support = True - try: - import jq - except ModuleNotFoundError: - jq_support = False - - watch = datastore.data['watching'].get(uuid) - - system_uses_webdriver = datastore.data['settings']['application']['fetch_backend'] == 'html_webdriver' - - watch_uses_webdriver = False - if (watch.get('fetch_backend') == 'system' and system_uses_webdriver) or watch.get('fetch_backend') == 'html_webdriver' or watch.get('fetch_backend', '').startswith('extra_browser_'): - watch_uses_webdriver = True - - from zoneinfo import available_timezones - - # Only works reliably with Playwright - - template_args = { - 'available_processors': processors.available_processors(), - 'available_timezones': sorted(available_timezones()), - 'browser_steps_config': browser_step_ui_config, - 'emailprefix': os.getenv('NOTIFICATION_MAIL_BUTTON_PREFIX', False), - 'extra_notification_token_placeholder_info': datastore.get_unique_notification_token_placeholders_available(), - 'extra_processor_config': form.extra_tab_content(), - 'extra_title': f" - Edit - {watch.label}", - 'form': form, - 'has_default_notification_urls': True if len(datastore.data['settings']['application']['notification_urls']) else False, - 'has_extra_headers_file': len(datastore.get_all_headers_in_textfile_for_watch(uuid=uuid)) > 0, - 'has_special_tag_options': _watch_has_tag_options_set(watch=watch), - 'watch_uses_webdriver': watch_uses_webdriver, - 'jq_support': jq_support, - 'playwright_enabled': os.getenv('PLAYWRIGHT_DRIVER_URL', False), - 'settings_application': datastore.data['settings']['application'], - 'timezone_default_config': datastore.data['settings']['application'].get('timezone'), - 'using_global_webdriver_wait': not default['webdriver_delay'], - 'uuid': uuid, - 'watch': watch - } - - included_content = None - if form.extra_form_content(): - # So that the extra panels can access _helpers.html etc, we set the environment to load from templates/ - # And then render the code from the module - from jinja2 import Environment, FileSystemLoader - import importlib.resources - templates_dir = str(importlib.resources.files("changedetectionio").joinpath('templates')) - env = Environment(loader=FileSystemLoader(templates_dir)) - template = env.from_string(form.extra_form_content()) - included_content = template.render(**template_args) - - output = render_template("edit.html", - extra_tab_content=form.extra_tab_content() if form.extra_tab_content() else None, - extra_form_content=included_content, - **template_args - ) - - return output @app.route("/import", methods=['GET', "POST"]) @@ -1117,41 +855,6 @@ def static_content(group, filename): except FileNotFoundError: abort(404) - @app.route("/edit//get-html", methods=['GET']) - @login_optionally_required - def watch_get_latest_html(uuid): - from io import BytesIO - from flask import send_file - import brotli - - watch = datastore.data['watching'].get(uuid) - if watch and watch.history.keys() and os.path.isdir(watch.watch_data_dir): - latest_filename = list(watch.history.keys())[-1] - html_fname = os.path.join(watch.watch_data_dir, f"{latest_filename}.html.br") - with open(html_fname, 'rb') as f: - if html_fname.endswith('.br'): - # Read and decompress the Brotli file - decompressed_data = brotli.decompress(f.read()) - else: - decompressed_data = f.read() - - buffer = BytesIO(decompressed_data) - - return send_file(buffer, as_attachment=True, download_name=f"{latest_filename}.html", mimetype='text/html') - - - # Return a 500 error - abort(500) - - # Ajax callback - @app.route("/edit//preview-rendered", methods=['POST']) - @login_optionally_required - def watch_get_preview_rendered(uuid): - '''For when viewing the "preview" of the rendered text from inside of Edit''' - from flask import jsonify - from .processors.text_json_diff import prepare_filter_prevew - result = prepare_filter_prevew(watch_uuid=uuid, form_data=request.form, datastore=datastore) - return jsonify(result) @app.route("/form/add/quickwatch", methods=['POST']) @@ -1243,26 +946,6 @@ def form_share_put_watch(): # paste in etc return redirect(url_for('index')) - @app.route("/highlight_submit_ignore_url", methods=['POST']) - @login_optionally_required - def highlight_submit_ignore_url(): - import re - mode = request.form.get('mode') - selection = request.form.get('selection') - - uuid = request.args.get('uuid','') - if datastore.data["watching"].get(uuid): - if mode == 'exact': - for l in selection.splitlines(): - datastore.data["watching"][uuid]['ignore_text'].append(l.strip()) - elif mode == 'digit-regex': - for l in selection.splitlines(): - # Replace any series of numbers with a regex - s = re.escape(l.strip()) - s = re.sub(r'[0-9]+', r'\\d+', s) - datastore.data["watching"][uuid]['ignore_text'].append('/' + s + '/') - - return f"Click to preview" import changedetectionio.blueprint.browser_steps as browser_steps diff --git a/changedetectionio/model/Watch.py b/changedetectionio/model/Watch.py index de9e8aa272c..c04af711bbb 100644 --- a/changedetectionio/model/Watch.py +++ b/changedetectionio/model/Watch.py @@ -83,7 +83,7 @@ def link(self): flash, Markup, url_for ) message = Markup('The URL {} is invalid and cannot be used, click to edit'.format( - url_for('edit_page', uuid=self.get('uuid')), self.get('url', ''))) + url_for('ui.ui_edit.edit_page', uuid=self.get('uuid')), self.get('url', ''))) flash(message, 'error') return '' diff --git a/changedetectionio/templates/base.html b/changedetectionio/templates/base.html index 59a834073aa..77d7520f0ea 100644 --- a/changedetectionio/templates/base.html +++ b/changedetectionio/templates/base.html @@ -74,7 +74,7 @@
    • {% else %}
  • - EDIT + EDIT
    • {% endif %} {% else %} diff --git a/changedetectionio/templates/diff.html b/changedetectionio/templates/diff.html index ecf05accf6c..a6301245b70 100644 --- a/changedetectionio/templates/diff.html +++ b/changedetectionio/templates/diff.html @@ -7,7 +7,7 @@ const error_screenshot_url="{{url_for('static_content', group='screenshot', filename=uuid, error_screenshot=1) }}"; {% endif %} - const highlight_submit_ignore_url="{{url_for('highlight_submit_ignore_url', uuid=uuid)}}"; + const highlight_submit_ignore_url="{{url_for('ui.ui_edit.highlight_submit_ignore_url', uuid=uuid)}}"; diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index 7b7be0f52fa..cb28db170b8 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -62,7 +62,7 @@
    + action="{{ url_for('ui.ui_edit.edit_page', uuid=uuid, next = request.args.get('next'), unpause_on_save = request.args.get('unpause_on_save'), tag = request.args.get('tag')) }}" method="POST">
    @@ -480,7 +480,7 @@

    Text filtering

    diff --git a/changedetectionio/templates/edit.html b/changedetectionio/templates/edit.html index cb28db170b8..3bda8e3fcf2 100644 --- a/changedetectionio/templates/edit.html +++ b/changedetectionio/templates/edit.html @@ -20,7 +20,7 @@ {% if emailprefix %} const email_notification_prefix=JSON.parse('{{ emailprefix|tojson }}'); {% endif %} - const notification_base_url="{{url_for('ajax_callback_send_notification_test', watch_uuid=uuid)}}"; + const notification_base_url="{{url_for('ui.ui_notification.ajax_callback_send_notification_test', watch_uuid=uuid)}}"; const playwright_enabled={% if playwright_enabled %}true{% else %}false{% endif %}; const recheck_proxy_start_url="{{url_for('check_proxies.start_check', uuid=uuid)}}"; const proxy_recheck_status_url="{{url_for('check_proxies.get_recheck_status', uuid=uuid)}}"; diff --git a/changedetectionio/templates/import.html b/changedetectionio/templates/import.html index 52d878de317..5b99b3b80ad 100644 --- a/changedetectionio/templates/import.html +++ b/changedetectionio/templates/import.html @@ -13,7 +13,7 @@
    - +
    diff --git a/changedetectionio/templates/settings.html b/changedetectionio/templates/settings.html index 0757867623c..2e651a0104e 100644 --- a/changedetectionio/templates/settings.html +++ b/changedetectionio/templates/settings.html @@ -4,7 +4,7 @@ {% from '_helpers.html' import render_field, render_checkbox_field, render_button, render_time_schedule_form %} {% from '_common_fields.html' import render_common_settings_form %}