Skip to content

Commit

Permalink
workaround for message is too big for the index (#50)
Browse files Browse the repository at this point in the history
  • Loading branch information
Szer authored Aug 27, 2024
1 parent f2b1c59 commit 2ab2a18
Showing 1 changed file with 8 additions and 2 deletions.
10 changes: 8 additions & 2 deletions src/VahterBanBot/DB.fs
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ open Dapper
open VahterBanBot.Utils

let private connString = getEnv "DATABASE_URL"
let private truncateTextLimit = getEnvOr "DATABASE_TRUNCATE_TEXT_LIMIT" "3000" |> int

let upsertUser (user: DbUser): Task<DbUser> =
task {
Expand Down Expand Up @@ -184,7 +185,7 @@ WITH less_than_n_messages AS (SELECT u.id, COUNT(DISTINCT m.text) < @criticalMsg
WHERE NOT EXISTS(SELECT 1 FROM false_positive_users fpu WHERE fpu.user_id = b.banned_user_id)
AND NOT EXISTS(SELECT 1
FROM false_positive_messages fpm
WHERE fpm.text = b.message_text)
WHERE b.message_text LIKE fpm.text || '%')
AND b.message_text IS NOT NULL
AND b.banned_at >= @criticalDate),
spam_or_ham AS (SELECT x.text,
Expand Down Expand Up @@ -246,6 +247,11 @@ WHERE banned_user_id = @userId
let markMessageAsFalsePositive (message: DbMessage): Task =
task {
use conn = new NpgsqlConnection(connString)

// we need to truncate the text to work around the fact that
// text is being indexed and btree index has a limit of 2704 bytes
// https://dba.stackexchange.com/questions/25138/index-max-row-size-error/25139
let message = { message with text = message.text.Substring(0, Math.Min(truncateTextLimit, message.text.Length)) }

//language=postgresql
let sql =
Expand Down Expand Up @@ -336,7 +342,7 @@ WITH stats AS (SELECT m.message_id,
ON m.message_id = bbb.message_id AND m.chat_id = bbb.banned_in_chat_id
LEFT JOIN public.false_negative_messages fnm
ON m.message_id = fnm.message_id AND m.chat_id = fnm.chat_id
LEFT JOIN false_positive_messages fpm ON m.text = fpm.text
LEFT JOIN false_positive_messages fpm ON m.text LIKE fpm.text || '%'
WHERE m.user_id = @userId
ORDER BY m.created_at DESC
LIMIT @n),
Expand Down

0 comments on commit 2ab2a18

Please sign in to comment.