Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

utils.ts example tweet splitting #323

Merged
merged 2 commits into from
Nov 15, 2024
Merged

utils.ts example tweet splitting #323

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
bf3ed27
utils.ts example tweet splitting
o-on-x Nov 15, 2024
46a913e
Merge branch 'ai16z:main' into o-on-x-wip-tweet-split
o-on-x Nov 15, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
212 changes: 93 additions & 119 deletions packages/client-twitter/src/utils.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
// utils.ts

import { Tweet } from "agent-twitter-client";
import { embeddingZeroVector } from "@ai16z/eliza/src/memory.ts";
import { Content, Memory, UUID } from "@ai16z/eliza/src/types.ts";
import { stringToUuid } from "@ai16z/eliza/src/uuid.ts";
import { ClientBase } from "./base.ts";
import { elizaLogger } from "@ai16z/eliza/src/logger.ts";

const MAX_TWEET_LENGTH = 240;
const MAX_TWEET_LENGTH = 280; // Updated to Twitter's current character limit

export const wait = (minTime: number = 1000, maxTime: number = 3000) => {
const waitTime =
Expand All @@ -17,13 +19,13 @@ export const isValidTweet = (tweet: Tweet): boolean => {
// Filter out tweets with too many hashtags, @s, or $ signs, probably spam or garbage
const hashtagCount = (tweet.text?.match(/#/g) || []).length;
const atCount = (tweet.text?.match(/@/g) || []).length;
const dollarSignCount = tweet.text?.match(/\$/g) || [];
const totalCount = hashtagCount + atCount + dollarSignCount.length;
const dollarSignCount = (tweet.text?.match(/\$/g) || []).length;
const totalCount = hashtagCount + atCount + dollarSignCount;

return (
hashtagCount <= 1 &&
atCount <= 2 &&
dollarSignCount.length <= 1 &&
dollarSignCount <= 1 &&
totalCount <= 3
);
};
Expand All @@ -40,7 +42,7 @@ export async function buildConversationThread(
elizaLogger.log("No current tweet found");
return;
}
// check if the current tweet has already been saved
// Check if the current tweet has already been saved
const memory = await client.runtime.messageManager.getMemoryById(
stringToUuid(currentTweet.id + "-" + client.runtime.agentId)
);
Expand All @@ -49,7 +51,10 @@ export async function buildConversationThread(
const roomId = stringToUuid(
currentTweet.conversationId + "-" + client.runtime.agentId
);
const userId = stringToUuid(currentTweet.userId);
const userId =
currentTweet.userId === client.twitterUserId
? client.runtime.agentId
: stringToUuid(currentTweet.userId);

await client.runtime.ensureConnection(
userId,
Expand All @@ -59,11 +64,12 @@ export async function buildConversationThread(
"twitter"
);

client.runtime.messageManager.createMemory({
await client.runtime.messageManager.createMemory({
id: stringToUuid(
currentTweet.id + "-" + client.runtime.agentId
),
agentId: client.runtime.agentId,
userId: userId,
content: {
text: currentTweet.text,
source: "twitter",
Expand All @@ -78,10 +84,6 @@ export async function buildConversationThread(
},
createdAt: currentTweet.timestamp * 1000,
roomId,
userId:
currentTweet.userId === client.twitterUserId
? client.runtime.agentId
: stringToUuid(currentTweet.userId),
embedding: embeddingZeroVector,
});
}
Expand All @@ -100,7 +102,7 @@ export async function buildConversationThread(
await processThread(tweet);
}

export async function sendTweetChunks(
export async function sendTweet(
client: ClientBase,
content: Content,
roomId: UUID,
Expand All @@ -109,25 +111,26 @@ export async function sendTweetChunks(
): Promise<Memory[]> {
const tweetChunks = splitTweetContent(content.text);
const sentTweets: Tweet[] = [];
let previousTweetId = inReplyTo;

for (const chunk of tweetChunks) {
const result = await client.requestQueue.add(
async () =>
await client.twitterClient.sendTweet(
chunk.replaceAll(/\\n/g, "\n").trim(),
inReplyTo
chunk.trim(),
previousTweetId
)
);
// console.log("send tweet result:\n", result);
// Parse the response
const body = await result.json();
console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
const tweetResult = body.data.create_tweet.tweet_results.result;

const finalTweet = {
const finalTweet: Tweet = {
id: tweetResult.rest_id,
text: tweetResult.legacy.full_text,
conversationId: tweetResult.legacy.conversation_id_str,
createdAt: tweetResult.legacy.created_at,
//createdAt:
timestamp: tweetResult.timestamp * 1000,
userId: tweetResult.legacy.user_id_str,
inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
Expand All @@ -137,72 +140,14 @@ export async function sendTweetChunks(
thread: [],
urls: [],
videos: [],
} as Tweet;
};

sentTweets.push(finalTweet);
}

const memories: Memory[] = sentTweets.map((tweet) => ({
id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
agentId: client.runtime.agentId,
userId: client.runtime.agentId,
content: {
text: tweet.text,
source: "twitter",
url: tweet.permanentUrl,
inReplyTo: tweet.inReplyToStatusId
? stringToUuid(
tweet.inReplyToStatusId + "-" + client.runtime.agentId
)
: undefined,
},
roomId,
embedding: embeddingZeroVector,
createdAt: tweet.timestamp * 1000,
}));

return memories;
}
previousTweetId = finalTweet.id;

export async function sendTweet(
client: ClientBase,
content: Content,
roomId: UUID,
twitterUsername: string,
inReplyTo: string
): Promise<Memory[]> {
const chunk = truncateTweetContent(content.text);
const sentTweets: Tweet[] = [];

const result = await client.requestQueue.add(
async () =>
await client.twitterClient.sendTweet(
chunk.replaceAll(/\\n/g, "\n").trim(),
inReplyTo
)
);
// console.log("send tweet result:\n", result);
const body = await result.json();
console.log("send tweet body:\n", body.data.create_tweet.tweet_results);
const tweetResult = body.data.create_tweet.tweet_results.result;

const finalTweet = {
id: tweetResult.rest_id,
text: tweetResult.legacy.full_text,
conversationId: tweetResult.legacy.conversation_id_str,
createdAt: tweetResult.legacy.created_at,
userId: tweetResult.legacy.user_id_str,
inReplyToStatusId: tweetResult.legacy.in_reply_to_status_id_str,
permanentUrl: `https://twitter.com/${twitterUsername}/status/${tweetResult.rest_id}`,
hashtags: [],
mentions: [],
photos: [],
thread: [],
urls: [],
videos: [],
} as Tweet;

sentTweets.push(finalTweet);
// Wait a bit between tweets to avoid rate limiting issues
await wait(1000, 2000);
}

const memories: Memory[] = sentTweets.map((tweet) => ({
id: stringToUuid(tweet.id + "-" + client.runtime.agentId),
Expand All @@ -227,56 +172,85 @@ export async function sendTweet(
}

function splitTweetContent(content: string): string[] {
const tweetChunks: string[] = [];
let currentChunk = "";
const maxLength = MAX_TWEET_LENGTH;
const paragraphs = content.split("\n\n").map((p) => p.trim());
const tweets: string[] = [];
let currentTweet = "";

for (const paragraph of paragraphs) {
if (!paragraph) continue;

const words = content.split(" ");
for (const word of words) {
if (currentChunk.length + word.length + 1 <= MAX_TWEET_LENGTH) {
currentChunk += (currentChunk ? " " : "") + word;
if ((currentTweet + "\n\n" + paragraph).trim().length <= maxLength) {
if (currentTweet) {
currentTweet += "\n\n" + paragraph;
} else {
currentTweet = paragraph;
}
} else {
tweetChunks.push(currentChunk);
currentChunk = word;
if (currentTweet) {
tweets.push(currentTweet.trim());
}
if (paragraph.length <= maxLength) {
currentTweet = paragraph;
} else {
// Split long paragraph into smaller chunks
const chunks = splitParagraph(paragraph, maxLength);
tweets.push(...chunks.slice(0, -1));
currentTweet = chunks[chunks.length - 1];
}
}
}

if (currentChunk) {
tweetChunks.push(currentChunk);
if (currentTweet) {
tweets.push(currentTweet.trim());
}

return tweetChunks;
return tweets;
}

export function truncateTweetContent(content: string): string {
// if its 240, delete the last line
if (content.length === MAX_TWEET_LENGTH) {
return content.slice(0, content.lastIndexOf("\n"));
}
function splitParagraph(paragraph: string, maxLength: number): string[] {
const sentences = paragraph.match(/[^\.!\?]+[\.!\?]+|[^\.!\?]+$/g) || [paragraph];
const chunks: string[] = [];
let currentChunk = "";

// if its still bigger than 240, delete everything after the last period
if (content.length > MAX_TWEET_LENGTH) {
return content.slice(0, content.lastIndexOf("."));
for (const sentence of sentences) {
if ((currentChunk + " " + sentence).trim().length <= maxLength) {
if (currentChunk) {
currentChunk += " " + sentence;
} else {
currentChunk = sentence;
}
} else {
if (currentChunk) {
chunks.push(currentChunk.trim());
}
if (sentence.length <= maxLength) {
currentChunk = sentence;
} else {
// Split long sentence into smaller pieces
const words = sentence.split(" ");
currentChunk = "";
for (const word of words) {
if ((currentChunk + " " + word).trim().length <= maxLength) {
if (currentChunk) {
currentChunk += " " + word;
} else {
currentChunk = word;
}
} else {
if (currentChunk) {
chunks.push(currentChunk.trim());
}
currentChunk = word;
}
}
}
}
}

// while its STILL bigger than 240, find the second to last exclamation point or period and delete everything after it
let iterations = 0;
while (content.length > MAX_TWEET_LENGTH && iterations < 10) {
iterations++;
// second to last index of period or exclamation point
const secondToLastIndexOfPeriod = content.lastIndexOf(
".",
content.length - 2
);
const secondToLastIndexOfExclamation = content.lastIndexOf(
"!",
content.length - 2
);
const secondToLastIndex = Math.max(
secondToLastIndexOfPeriod,
secondToLastIndexOfExclamation
);
content = content.slice(0, secondToLastIndex);
if (currentChunk) {
chunks.push(currentChunk.trim());
}

return content;
return chunks;
}
Loading