chore: normalize urls extraction for messages

apps/meteor/app/lib/server/functions/extractUrlsFromMessageAST.spec.ts

@@ -0,0 +1,115 @@
+import { expect } from 'chai';
+import { describe, it } from 'mocha';
+
+import { extractUrlsFromMessageAST } from './extractUrlsFromMessageAST';
+
+describe('extractUrlsFromMessageAST', () => {
+	it('should extract URLs from LINK nodes', () => {
+		const md = [
+			{
+				type: 'PARAGRAPH',
+				value: [
+					{
+						type: 'LINK',
+						value: {
+							src: {
+								type: 'PLAIN_TEXT',
+								value: 'https://rocket.chat',
+							},
+							label: [
+								{
+									type: 'PLAIN_TEXT',
+									value: 'rocket.chat',
+								},
+							],
+						},
+					},
+				],
+			},
+		];
+
+		const urls = extractUrlsFromMessageAST(md as any);
+		expect(urls).to.deep.equal(['https://rocket.chat']);
+	});
+
+	it('should convert // prefix to https://', () => {
+		const md = [
+			{
+				type: 'PARAGRAPH',
+				value: [
+					{
+						type: 'LINK',
+						value: {
+							src: {
+								type: 'PLAIN_TEXT',
+								value: '//github.com/RocketChat/Rocket.Chat',
+							},
+							label: [
+								{
+									type: 'PLAIN_TEXT',
+									value: 'github.com/RocketChat/Rocket.Chat',
+								},
+							],
+						},
+					},
+				],
+			},
+		];
+
+		const urls = extractUrlsFromMessageAST(md as any);
+		expect(urls).to.deep.equal(['https://github.com/RocketChat/Rocket.Chat']);
+	});
+
+	it('should handle multiple links', () => {
+		const md = [
+			{
+				type: 'PARAGRAPH',
+				value: [
+					{
+						type: 'LINK',
+						value: {
+							src: {
+								type: 'PLAIN_TEXT',
+								value: 'https://rocket.chat',
+							},
+							label: [
+								{
+									type: 'PLAIN_TEXT',
+									value: 'rocket.chat',
+								},
+							],
+						},
+					},
+					{
+						type: 'PLAIN_TEXT',
+						value: ' and ',
+					},
+					{
+						type: 'LINK',
+						value: {
+							src: {
+								type: 'PLAIN_TEXT',
+								value: '//github.com/RocketChat',
+							},
+							label: [
+								{
+									type: 'PLAIN_TEXT',
+									value: 'github.com/RocketChat',
+								},
+							],
+						},
+					},
+				],
+			},
+		];
+
+		const urls = extractUrlsFromMessageAST(md as any);
+		expect(urls).to.deep.equal(['https://rocket.chat', 'https://github.com/RocketChat']);
+	});
+
+	it('should return empty array for undefined or non-array input', () => {
+		expect(extractUrlsFromMessageAST(undefined)).to.deep.equal([]);
+		expect(extractUrlsFromMessageAST(null as any)).to.deep.equal([]);
+		expect(extractUrlsFromMessageAST({} as any)).to.deep.equal([]);
+	});
+});

apps/meteor/app/lib/server/functions/extractUrlsFromMessageAST.ts

@@ -0,0 +1,33 @@
+import type { Root } from '@rocket.chat/message-parser';
+
+/**
+ * Extracts all URLs from parsed message AST (message-parser output)
+ * Looks for LINK nodes and extracts the src URL
+ */
+export const extractUrlsFromMessageAST = (md?: Root | Root[number] | Root[number]['value']): string[] => {
+	if (!md || !Array.isArray(md)) {
+		return [];
+	}
+
+	const urls: string[] = [];
+
+	const walk = (node: any): void => {
+		if (Array.isArray(node)) {
+			node.forEach(walk);
+			return;
+		}
+		if (typeof node !== 'object' || node === null) {
+			return;
+		}
+		if (node.type === 'LINK' && node.value?.src?.value) {
+			urls.push(node.value.src.value);
+		}
+		if (node.value !== undefined) {
+			walk(node.value);
+		}
+	};
+
+	walk(md);
+
+	return urls;
+};

apps/meteor/app/lib/server/functions/insertMessage.ts

@@ -4,6 +4,7 @@ import { Messages, Rooms } from '@rocket.chat/models';
 import { parseUrlsInMessage } from './parseUrlsInMessage';
 import { validateMessage, prepareMessageObject } from './sendMessage';
 
+// TODO: remove and move to Message.Service
 export const insertMessage = async function (
 	user: Pick<IUser, '_id' | 'username'>,
 	message: IMessage,
@@ -16,7 +17,7 @@ export const insertMessage = async function (
 
 	await validateMessage(message, { _id: rid }, user);
 	prepareMessageObject(message, rid, user);
-	parseUrlsInMessage(message);
+	message.urls = parseUrlsInMessage(message);
 
 	if (message._id && upsert) {
 		const { _id, ...rest } = message;

apps/meteor/app/lib/server/functions/parseUrlsInMessage.ts

@@ -1,29 +1,37 @@
 import type { IMessage, AtLeast } from '@rocket.chat/core-typings';
 
+import { extractUrlsFromMessageAST } from './extractUrlsFromMessageAST';
 import { getMessageUrlRegex } from '../../../../lib/getMessageUrlRegex';
 import { Markdown } from '../../../markdown/server';
 import { settings } from '../../../settings/server';
 
-// TODO move this function to message service to be used like a "beforeSaveMessage" hook
-export const parseUrlsInMessage = (message: AtLeast<IMessage, 'msg'> & { parseUrls?: boolean }, previewUrls?: string[]) => {
-	if (message.parseUrls === false) {
-		return message;
-	}
+const prepareUrl = (url: string, previewUrls: string[] | undefined) => ({
+	url,
+	meta: {},
+	...(previewUrls && !previewUrls.includes(url) && !url.includes(settings.get('Site_Url')) && { ignoreParse: true }),
+});
 
-	message.html = message.msg;
-	message = Markdown.code(message);
+const prepareUrls = (urls: string[], previewUrls?: string[]) => [...new Set(urls)].map((url) => prepareUrl(url, previewUrls));
 
-	const urls = message.html?.match(getMessageUrlRegex()) || [];
-	if (urls) {
-		message.urls = [...new Set(urls)].map((url) => ({
-			url,
-			meta: {},
-			...(previewUrls && !previewUrls.includes(url) && !url.includes(settings.get('Site_Url')) && { ignoreParse: true }),
-		}));
+export const parseUrlsInMessage = (
+	message: AtLeast<IMessage, 'msg' | 'md'> & {
+		parseUrls?: boolean;
+	},
+	previewUrls?: string[],
+) => {
+	// Also extract URLs from message blocks if they exist
+	if (message.md) {
+		const astUrls = extractUrlsFromMessageAST(message.md);
+		return prepareUrls(astUrls, previewUrls);
 	}
 
-	message = Markdown.mountTokensBack(message, false);
-	message.msg = message.html || message.msg;
-	delete message.html;
-	delete message.tokens;
+	// TODO: remove this after make the parser official
+	// Parse the message to extract URLs from links without schema
+	// The message parser converts links like "github.com" to proper links with "//" prefix
+	const result = Markdown.code({
+		html: message.msg,
+		msg: message.msg,
+	});
+	const htmlUrls = result.html?.match(getMessageUrlRegex()) || [];
+	return prepareUrls(htmlUrls, previewUrls);
 };

apps/meteor/app/lib/server/functions/sendMessage.ts

@@ -4,7 +4,6 @@ import type { IMessage, IRoom } from '@rocket.chat/core-typings';
 import { Messages } from '@rocket.chat/models';
 import { Match, check } from 'meteor/check';
 
-import { parseUrlsInMessage } from './parseUrlsInMessage';
 import { isRelativeURL } from '../../../../lib/utils/isRelativeURL';
 import { isURL } from '../../../../lib/utils/isURL';
 import { hasPermissionAsync } from '../../../authorization/server/functions/hasPermission';
@@ -257,9 +256,7 @@ export const sendMessage = async function (user: any, message: any, room: any, o
 		}
 	}
 
-	parseUrlsInMessage(message, previewUrls);
-
-	message = await Message.beforeSave({ message, room, user });
+	message = await Message.beforeSave({ message, room, user, previewUrls, parseUrls: message.parseUrls });
 
 	if (!message) {
 		return;

apps/meteor/app/lib/server/functions/updateMessage.ts

@@ -4,14 +4,18 @@ import type { IMessage, IUser, AtLeast } from '@rocket.chat/core-typings';
 import { Messages, Rooms } from '@rocket.chat/models';
 import { Meteor } from 'meteor/meteor';
 
-import { parseUrlsInMessage } from './parseUrlsInMessage';
 import { settings } from '../../../settings/server';
 import { afterSaveMessage } from '../lib/afterSaveMessage';
 import { notifyOnRoomChangedById } from '../lib/notifyListener';
 import { validateCustomMessageFields } from '../lib/validateCustomMessageFields';
 
 export const updateMessage = async function (
-	message: AtLeast<IMessage, '_id' | 'rid' | 'msg' | 'customFields'> | AtLeast<IMessage, '_id' | 'rid' | 'content'>,
+	{
+		parseUrls,
+		...message
+	}: (AtLeast<IMessage, '_id' | 'rid' | 'msg' | 'customFields'> | AtLeast<IMessage, '_id' | 'rid' | 'content'>) & {
+		parseUrls?: boolean;
+	},
 	user: IUser,
 	originalMsg?: IMessage,
 	previewUrls?: string[],
@@ -51,14 +55,12 @@ export const updateMessage = async function (
 		},
 	});
 
-	parseUrlsInMessage(messageData, previewUrls);
-
 	const room = await Rooms.findOneById(messageData.rid);
 	if (!room) {
 		return;
 	}
 
-	messageData = await Message.beforeSave({ message: messageData, room, user });
+	messageData = await Message.beforeSave({ message: messageData, room, user, previewUrls, parseUrls });
 
 	if (messageData.customFields) {
 		validateCustomMessageFields({

apps/meteor/app/markdown/lib/parser/original/token.ts

@@ -2,10 +2,19 @@
  * Markdown is a named function that will parse markdown syntax
  * @param {String} msg - The message html
  */
-import type { IMessage, TokenType, TokenExtra } from '@rocket.chat/core-typings';
+import type { TokenType, TokenExtra } from '@rocket.chat/core-typings';
 import { Random } from '@rocket.chat/random';
 
-export const addAsToken = (message: IMessage, html: string, type: TokenType, extra?: TokenExtra): string => {
+type MessageTokens = {
+	tokens?: {
+		token: string;
+		type: TokenType;
+		text: string;
+		extra?: TokenExtra;
+	}[];
+};
+
+export const addAsToken = (message: MessageTokens, html: string, type: TokenType, extra?: TokenExtra): string => {
 	if (!message.tokens) {
 		message.tokens = [];
 	}
@@ -22,7 +31,7 @@ export const addAsToken = (message: IMessage, html: string, type: TokenType, ext
 
 export const isToken = (msg: string): boolean => /=!=[.a-z0-9]{17}=!=/gim.test(msg.trim());
 
-export const validateAllowedTokens = (message: IMessage, id: string, desiredTokens: TokenType[]): boolean => {
+export const validateAllowedTokens = (message: MessageTokens, id: string, desiredTokens: TokenType[]): boolean => {
 	const tokens: string[] = id.match(/=!=[.a-z0-9]{17}=!=/gim) || [];
 	const tokensFound = message.tokens?.filter(({ token }) => tokens.includes(token)) || [];
 	return tokensFound.length === 0 || tokensFound.every((token) => token.type && desiredTokens.includes(token.type));

apps/meteor/server/services/messages/service.ts

@@ -7,6 +7,7 @@ import { Messages, Rooms } from '@rocket.chat/models';
 
 import { OEmbed } from './hooks/AfterSaveOEmbed';
 import { deleteMessage } from '../../../app/lib/server/functions/deleteMessage';
+import { parseUrlsInMessage } from '../../../app/lib/server/functions/parseUrlsInMessage';
 import { sendMessage } from '../../../app/lib/server/functions/sendMessage';
 import { updateMessage } from '../../../app/lib/server/functions/updateMessage';
 import { notifyOnRoomChangedById, notifyOnMessageChange } from '../../../app/lib/server/lib/notifyListener';
@@ -217,10 +218,14 @@ export class MessageService extends ServiceClassInternal implements IMessageServ
 		message,
 		room,
 		user,
+		previewUrls,
+		parseUrls = true,
 	}: {
 		message: IMessage;
 		room: IRoom;
 		user: Pick<IUser, '_id' | 'username' | 'name' | 'emails' | 'language'>;
+		previewUrls?: string[];
+		parseUrls?: boolean;
 	}): Promise<IMessage> {
 		// TODO looks like this one was not being used (so I'll left it commented)
 		// await this.joinDiscussionOnMessage({ message, room, user });
@@ -233,6 +238,9 @@ export class MessageService extends ServiceClassInternal implements IMessageServ
 		message = await this.cannedResponse.replacePlaceholders({ message, room, user });
 		message = await this.badWords.filterBadWords({ message });
 		message = await this.markdownParser.parseMarkdown({ message, config: this.getMarkdownConfig() });
+		if (parseUrls) {
+			message.urls = parseUrlsInMessage(message, previewUrls);
+		}
 		message = await this.spotify.convertSpotifyLinks({ message });
 		message = await this.jumpToMessage.createAttachmentForMessageURLs({
 			message,