From bf3ed27015bcabe89492540de23f9c9cab61f833 Mon Sep 17 00:00:00 2001 From: o-on-x Date: Thu, 14 Nov 2024 19:26:00 -0600 Subject: [PATCH] utils.ts example tweet splitting if no other splitting is done in the other modules would work interactions.ts & post.ts should then just generate any size no character limit tweet (within reason) wip --- packages/client-twitter/src/utils.ts | 212 ++++++++++++--------------- 1 file changed, 93 insertions(+), 119 deletions(-) diff --git a/packages/client-twitter/src/utils.ts b/packages/client-twitter/src/utils.ts index b00e2554fdb..cc95739ceae 100644 --- a/packages/client-twitter/src/utils.ts +++ b/packages/client-twitter/src/utils.ts @@ -1,3 +1,5 @@ +// utils.ts + import { Tweet } from "agent-twitter-client"; import { embeddingZeroVector } from "@ai16z/eliza/src/memory.ts"; import { Content, Memory, UUID } from "@ai16z/eliza/src/types.ts"; @@ -5,7 +7,7 @@ import { stringToUuid } from "@ai16z/eliza/src/uuid.ts"; import { ClientBase } from "./base.ts"; import { elizaLogger } from "@ai16z/eliza/src/logger.ts"; -const MAX_TWEET_LENGTH = 240; +const MAX_TWEET_LENGTH = 280; // Updated to Twitter's current character limit export const wait = (minTime: number = 1000, maxTime: number = 3000) => { const waitTime = @@ -17,13 +19,13 @@ export const isValidTweet = (tweet: Tweet): boolean => { // Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage const hashtagCount = (tweet.text?.match(/#/g) || []).length; const atCount = (tweet.text?.match(/@/g) || []).length; - const dollarSignCount = tweet.text?.match(/\$/g) || []; - const totalCount = hashtagCount + atCount + dollarSignCount.length; + const dollarSignCount = (tweet.text?.match(/\$/g) || []).length; + const totalCount = hashtagCount + atCount + dollarSignCount; return ( hashtagCount <= 1 && atCount <= 2 && - dollarSignCount.length <= 1 && + dollarSignCount <= 1 && totalCount <= 3 ); }; @@ -40,7 +42,7 @@ export async function buildConversationThread( elizaLogger.log("No current tweet found"); return; } - // check if the current tweet has already been saved + // Check if the current tweet has already been saved const memory = await client.runtime.messageManager.getMemoryById( stringToUuid(currentTweet.id + "-" + client.runtime.agentId) ); @@ -49,7 +51,10 @@ export async function buildConversationThread( const roomId = stringToUuid( currentTweet.conversationId + "-" + client.runtime.agentId ); - const userId = stringToUuid(currentTweet.userId); + const userId = + currentTweet.userId === client.twitterUserId + ? client.runtime.agentId + : stringToUuid(currentTweet.userId); await client.runtime.ensureConnection( userId, @@ -59,11 +64,12 @@ export async function buildConversationThread( "twitter" ); - client.runtime.messageManager.createMemory({ + await client.runtime.messageManager.createMemory({ id: stringToUuid( currentTweet.id + "-" + client.runtime.agentId ), agentId: client.runtime.agentId, + userId: userId, content: { text: currentTweet.text, source: "twitter", @@ -78,10 +84,6 @@ export async function buildConversationThread( }, createdAt: currentTweet.timestamp * 1000, roomId, - userId: - currentTweet.userId === client.twitterUserId - ? client.runtime.agentId - : stringToUuid(currentTweet.userId), embedding: embeddingZeroVector, }); } @@ -100,7 +102,7 @@ export async function buildConversationThread( await processThread(tweet); } -export async function sendTweetChunks( +export async function sendTweet( client: ClientBase, content: Content, roomId: UUID, @@ -109,25 +111,26 @@ export async function sendTweetChunks( ): Promise { const tweetChunks = splitTweetContent(content.text); const sentTweets: Tweet[] = []; + let previousTweetId = inReplyTo; for (const chunk of tweetChunks) { const result = await client.requestQueue.add( async () => await client.twitterClient.sendTweet( - chunk.replaceAll(/\\n/g, "\n").trim(), - inReplyTo + chunk.trim(), + previousTweetId ) ); - // console.log("send tweet result:\n", result); + // Parse the response const body = await result.json(); - console.log("send tweet body:\n", body.data.create_tweet.tweet_results); const tweetResult = body.data.create_tweet.tweet_results.result; - const finalTweet = { + const finalTweet: Tweet = { id: tweetResult.rest_id, text: tweetResult.legacy.full_text, conversationId: tweetResult.legacy.conversation_id_str, - createdAt: tweetResult.legacy.created_at, + //createdAt: + timestamp: tweetResult.timestamp * 1000, userId: tweetResult.legacy.user_id_str, inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str, permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`, @@ -137,72 +140,14 @@ export async function sendTweetChunks( thread: [], urls: [], videos: [], - } as Tweet; + }; sentTweets.push(finalTweet); - } - - const memories: Memory[] = sentTweets.map((tweet) => ({ - id: stringToUuid(tweet.id + "-" + client.runtime.agentId), - agentId: client.runtime.agentId, - userId: client.runtime.agentId, - content: { - text: tweet.text, - source: "twitter", - url: tweet.permanentUrl, - inReplyTo: tweet.inReplyToStatusId - ? stringToUuid( - tweet.inReplyToStatusId + "-" + client.runtime.agentId - ) - : undefined, - }, - roomId, - embedding: embeddingZeroVector, - createdAt: tweet.timestamp * 1000, - })); - - return memories; -} + previousTweetId = finalTweet.id; -export async function sendTweet( - client: ClientBase, - content: Content, - roomId: UUID, - twitterUsername: string, - inReplyTo: string -): Promise { - const chunk = truncateTweetContent(content.text); - const sentTweets: Tweet[] = []; - - const result = await client.requestQueue.add( - async () => - await client.twitterClient.sendTweet( - chunk.replaceAll(/\\n/g, "\n").trim(), - inReplyTo - ) - ); - // console.log("send tweet result:\n", result); - const body = await result.json(); - console.log("send tweet body:\n", body.data.create_tweet.tweet_results); - const tweetResult = body.data.create_tweet.tweet_results.result; - - const finalTweet = { - id: tweetResult.rest_id, - text: tweetResult.legacy.full_text, - conversationId: tweetResult.legacy.conversation_id_str, - createdAt: tweetResult.legacy.created_at, - userId: tweetResult.legacy.user_id_str, - inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str, - permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`, - hashtags: [], - mentions: [], - photos: [], - thread: [], - urls: [], - videos: [], - } as Tweet; - - sentTweets.push(finalTweet); + // Wait a bit between tweets to avoid rate limiting issues + await wait(1000, 2000); + } const memories: Memory[] = sentTweets.map((tweet) => ({ id: stringToUuid(tweet.id + "-" + client.runtime.agentId), @@ -227,56 +172,85 @@ export async function sendTweet( } function splitTweetContent(content: string): string[] { - const tweetChunks: string[] = []; - let currentChunk = ""; + const maxLength = MAX_TWEET_LENGTH; + const paragraphs = content.split("\n\n").map((p) => p.trim()); + const tweets: string[] = []; + let currentTweet = ""; + + for (const paragraph of paragraphs) { + if (!paragraph) continue; - const words = content.split(" "); - for (const word of words) { - if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) { - currentChunk += (currentChunk ? " " : "") + word; + if ((currentTweet + "\n\n" + paragraph).trim().length <= maxLength) { + if (currentTweet) { + currentTweet += "\n\n" + paragraph; + } else { + currentTweet = paragraph; + } } else { - tweetChunks.push(currentChunk); - currentChunk = word; + if (currentTweet) { + tweets.push(currentTweet.trim()); + } + if (paragraph.length <= maxLength) { + currentTweet = paragraph; + } else { + // Split long paragraph into smaller chunks + const chunks = splitParagraph(paragraph, maxLength); + tweets.push(...chunks.slice(0, -1)); + currentTweet = chunks[chunks.length - 1]; + } } } - if (currentChunk) { - tweetChunks.push(currentChunk); + if (currentTweet) { + tweets.push(currentTweet.trim()); } - return tweetChunks; + return tweets; } -export function truncateTweetContent(content: string): string { - // if its 240, delete the last line - if (content.length === MAX_TWEET_LENGTH) { - return content.slice(0, content.lastIndexOf("\n")); - } +function splitParagraph(paragraph: string, maxLength: number): string[] { + const sentences = paragraph.match(/[^\.!\?]+[\.!\?]+|[^\.!\?]+$/g) || [paragraph]; + const chunks: string[] = []; + let currentChunk = ""; - // if its still bigger than 240, delete everything after the last period - if (content.length > MAX_TWEET_LENGTH) { - return content.slice(0, content.lastIndexOf(".")); + for (const sentence of sentences) { + if ((currentChunk + " " + sentence).trim().length <= maxLength) { + if (currentChunk) { + currentChunk += " " + sentence; + } else { + currentChunk = sentence; + } + } else { + if (currentChunk) { + chunks.push(currentChunk.trim()); + } + if (sentence.length <= maxLength) { + currentChunk = sentence; + } else { + // Split long sentence into smaller pieces + const words = sentence.split(" "); + currentChunk = ""; + for (const word of words) { + if ((currentChunk + " " + word).trim().length <= maxLength) { + if (currentChunk) { + currentChunk += " " + word; + } else { + currentChunk = word; + } + } else { + if (currentChunk) { + chunks.push(currentChunk.trim()); + } + currentChunk = word; + } + } + } + } } - // while its STILL bigger than 240, find the second to last exclamation point or period and delete everything after it - let iterations = 0; - while (content.length > MAX_TWEET_LENGTH && iterations < 10) { - iterations++; - // second to last index of period or exclamation point - const secondToLastIndexOfPeriod = content.lastIndexOf( - ".", - content.length - 2 - ); - const secondToLastIndexOfExclamation = content.lastIndexOf( - "!", - content.length - 2 - ); - const secondToLastIndex = Math.max( - secondToLastIndexOfPeriod, - secondToLastIndexOfExclamation - ); - content = content.slice(0, secondToLastIndex); + if (currentChunk) { + chunks.push(currentChunk.trim()); } - return content; + return chunks; }