Skip to content

Commit

Permalink
Merge pull request #323 from o-on-x/o-on-x-wip-tweet-split
Browse files Browse the repository at this point in the history 
utils.ts example tweet splitting
  • Loading branch information
@lalalune
lalalune authored Nov 15, 2024
2 parents 0d1b1fb + 46a913e commit 1824bb9
Showing 1 changed file with 93 additions and 119 deletions.
212 changes: 93 additions & 119 deletions packages/client-twitter/src/utils.ts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
// utils.ts

import { Tweet } from "agent-twitter-client";
import { embeddingZeroVector } from "@ai16z/eliza/src/memory.ts";
import { Content, Memory, UUID } from "@ai16z/eliza/src/types.ts";
import { stringToUuid } from "@ai16z/eliza/src/uuid.ts";
import { ClientBase } from "./base.ts";
import { elizaLogger } from "@ai16z/eliza/src/logger.ts";

const MAX_TWEET_LENGTH = 240;
const MAX_TWEET_LENGTH = 280; // Updated to Twitter's current character limit

export const wait = (minTime: number = 1000, maxTime: number = 3000) => {
const waitTime =
Expand All @@ -17,13 +19,13 @@ export const isValidTweet = (tweet: Tweet): boolean => {
// Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage
const hashtagCount = (tweet.text?.match(/#/g) || []).length;
const atCount = (tweet.text?.match(/@/g) || []).length;
const dollarSignCount = tweet.text?.match(/\$/g) || [];
const totalCount = hashtagCount + atCount + dollarSignCount.length;
const dollarSignCount = (tweet.text?.match(/\$/g) || []).length;
const totalCount = hashtagCount + atCount + dollarSignCount;

return (
hashtagCount <= 1 &&
atCount <= 2 &&
dollarSignCount.length <= 1 &&
dollarSignCount <= 1 &&
totalCount <= 3
);
};
Expand All @@ -40,7 +42,7 @@ export async function buildConversationThread(
elizaLogger.log("No current tweet found");
return;
}
// check if the current tweet has already been saved
// Check if the current tweet has already been saved
const memory = await client.runtime.messageManager.getMemoryById(
stringToUuid(currentTweet.id + "-" + client.runtime.agentId)
);
Expand All @@ -49,7 +51,10 @@ export async function buildConversationThread(
const roomId = stringToUuid(
currentTweet.conversationId + "-" + client.runtime.agentId
);
const userId = stringToUuid(currentTweet.userId);
const userId =
currentTweet.userId === client.twitterUserId
? client.runtime.agentId
: stringToUuid(currentTweet.userId);

await client.runtime.ensureConnection(
userId,
Expand All @@ -59,11 +64,12 @@ export async function buildConversationThread(
"twitter"
);

client.runtime.messageManager.createMemory({
await client.runtime.messageManager.createMemory({
id: stringToUuid(
currentTweet.id + "-" + client.runtime.agentId
),
agentId: client.runtime.agentId,
userId: userId,
content: {
text: currentTweet.text,
source: "twitter",
Expand All @@ -78,10 +84,6 @@ export async function buildConversationThread(
},
createdAt: currentTweet.timestamp * 1000,
roomId,
userId:
currentTweet.userId === client.twitterUserId
? client.runtime.agentId
: stringToUuid(currentTweet.userId),
embedding: embeddingZeroVector,
});
}
Expand All @@ -100,7 +102,7 @@ export async function buildConversationThread(
await processThread(tweet);
}

export async function sendTweetChunks(
export async function sendTweet(
client: ClientBase,
content: Content,
roomId: UUID,
Expand All @@ -109,25 +111,26 @@ export async function sendTweetChunks(
): Promise<Memory[]> {
const tweetChunks = splitTweetContent(content.text);
const sentTweets: Tweet[] = [];
let previousTweetId = inReplyTo;

for (const chunk of tweetChunks) {
const result = await client.requestQueue.add(
async () =>
await client.twitterClient.sendTweet(
chunk.replaceAll(/\\n/g, "\n").trim(),
inReplyTo
chunk.trim(),
previousTweetId
)
);
// console.log("send tweet result:\n", result);
// Parse the response
const body = await result.json();
console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
const tweetResult = body.data.create_tweet.tweet_results.result;

const finalTweet = {
const finalTweet: Tweet = {
id: tweetResult.rest_id,
text: tweetResult.legacy.full_text,
conversationId: tweetResult.legacy.conversation_id_str,
createdAt: tweetResult.legacy.created_at,
//createdAt:
timestamp: tweetResult.timestamp * 1000,
userId: tweetResult.legacy.user_id_str,
inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
Expand All @@ -137,72 +140,14 @@ export async function sendTweetChunks(
thread: [],
urls: [],
videos: [],
} as Tweet;
};

sentTweets.push(finalTweet);
}

const memories: Memory[] = sentTweets.map((tweet) => ({
id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
agentId: client.runtime.agentId,
userId: client.runtime.agentId,
content: {
text: tweet.text,
source: "twitter",
url: tweet.permanentUrl,
inReplyTo: tweet.inReplyToStatusId
? stringToUuid(
tweet.inReplyToStatusId + "-" + client.runtime.agentId
)
: undefined,
},
roomId,
embedding: embeddingZeroVector,
createdAt: tweet.timestamp * 1000,
}));

return memories;
}
previousTweetId = finalTweet.id;

export async function sendTweet(
client: ClientBase,
content: Content,
roomId: UUID,
twitterUsername: string,
inReplyTo: string
): Promise<Memory[]> {
const chunk = truncateTweetContent(content.text);
const sentTweets: Tweet[] = [];

const result = await client.requestQueue.add(
async () =>
await client.twitterClient.sendTweet(
chunk.replaceAll(/\\n/g, "\n").trim(),
inReplyTo
)
);
// console.log("send tweet result:\n", result);
const body = await result.json();
console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
const tweetResult = body.data.create_tweet.tweet_results.result;

const finalTweet = {
id: tweetResult.rest_id,
text: tweetResult.legacy.full_text,
conversationId: tweetResult.legacy.conversation_id_str,
createdAt: tweetResult.legacy.created_at,
userId: tweetResult.legacy.user_id_str,
inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
hashtags: [],
mentions: [],
photos: [],
thread: [],
urls: [],
videos: [],
} as Tweet;

sentTweets.push(finalTweet);
// Wait a bit between tweets to avoid rate limiting issues
await wait(1000, 2000);
}

const memories: Memory[] = sentTweets.map((tweet) => ({
id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
Expand All @@ -227,56 +172,85 @@ export async function sendTweet(
}

function splitTweetContent(content: string): string[] {
const tweetChunks: string[] = [];
let currentChunk = "";
const maxLength = MAX_TWEET_LENGTH;
const paragraphs = content.split("\n\n").map((p) => p.trim());
const tweets: string[] = [];
let currentTweet = "";

for (const paragraph of paragraphs) {
if (!paragraph) continue;

const words = content.split(" ");
for (const word of words) {
if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) {
currentChunk += (currentChunk ? " " : "") + word;
if ((currentTweet + "\n\n" + paragraph).trim().length <= maxLength) {
if (currentTweet) {
currentTweet += "\n\n" + paragraph;
} else {
currentTweet = paragraph;
}
} else {
tweetChunks.push(currentChunk);
currentChunk = word;
if (currentTweet) {
tweets.push(currentTweet.trim());
}
if (paragraph.length <= maxLength) {
currentTweet = paragraph;
} else {
// Split long paragraph into smaller chunks
const chunks = splitParagraph(paragraph, maxLength);
tweets.push(...chunks.slice(0, -1));
currentTweet = chunks[chunks.length - 1];
}
}
}

if (currentChunk) {
tweetChunks.push(currentChunk);
if (currentTweet) {
tweets.push(currentTweet.trim());
}

return tweetChunks;
return tweets;
}

export function truncateTweetContent(content: string): string {
// if its 240, delete the last line
if (content.length === MAX_TWEET_LENGTH) {
return content.slice(0, content.lastIndexOf("\n"));
}
function splitParagraph(paragraph: string, maxLength: number): string[] {
const sentences = paragraph.match(/[^\.!\?]+[\.!\?]+|[^\.!\?]+$/g) || [paragraph];
const chunks: string[] = [];
let currentChunk = "";

// if its still bigger than 240, delete everything after the last period
if (content.length > MAX_TWEET_LENGTH) {
return content.slice(0, content.lastIndexOf("."));
for (const sentence of sentences) {
if ((currentChunk + " " + sentence).trim().length <= maxLength) {
if (currentChunk) {
currentChunk += " " + sentence;
} else {
currentChunk = sentence;
}
} else {
if (currentChunk) {
chunks.push(currentChunk.trim());
}
if (sentence.length <= maxLength) {
currentChunk = sentence;
} else {
// Split long sentence into smaller pieces
const words = sentence.split(" ");
currentChunk = "";
for (const word of words) {
if ((currentChunk + " " + word).trim().length <= maxLength) {
if (currentChunk) {
currentChunk += " " + word;
} else {
currentChunk = word;
}
} else {
if (currentChunk) {
chunks.push(currentChunk.trim());
}
currentChunk = word;
}
}
}
}
}

// while its STILL bigger than 240, find the second to last exclamation point or period and delete everything after it
let iterations = 0;
while (content.length > MAX_TWEET_LENGTH && iterations < 10) {
iterations++;
// second to last index of period or exclamation point
const secondToLastIndexOfPeriod = content.lastIndexOf(
".",
content.length - 2
);
const secondToLastIndexOfExclamation = content.lastIndexOf(
"!",
content.length - 2
);
const secondToLastIndex = Math.max(
secondToLastIndexOfPeriod,
secondToLastIndexOfExclamation
);
content = content.slice(0, secondToLastIndex);
if (currentChunk) {
chunks.push(currentChunk.trim());
}

return content;
return chunks;
}

0 comments on commit 1824bb9

Please sign in to comment.