Improve Per-extractor Instructions

.vscode/launch.json

@@ -0,0 +1,13 @@
+{
+  "version": "0.2.0",
+  "configurations": [
+    {
+      "name": "Local",
+      "type": "coreclr",
+      "request": "launch",
+      "program": "${workspaceFolder}/src/Elzik.Breef.Api/bin/Debug/net8.0/Elzik.Breef.Api.dll",
+      "cwd": "${workspaceFolder}/src/Elzik.Breef.Api",
+      "stopAtEntry": false
+    }
+  ]
+}

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/Raw/RawRedditPostTransformer.cs

@@ -5,41 +5,6 @@
 
 public class RawRedditPostTransformer : IRawRedditPostTransformer
 {
-    public RedditPost Transform(RawRedditPost rawRedditPost)
-    {
-        ArgumentNullException.ThrowIfNull(rawRedditPost);
-        if (rawRedditPost.Count < 2)
-            throw new ArgumentException("Reddit post must have at least 2 listings (post and comments)", nameof(rawRedditPost));
-
-        var postListing = rawRedditPost[0];
-        var commentsListing = rawRedditPost[1];
-
-        var postChildren = postListing.Data?.Children;
-        if (postChildren == null || postChildren.Count == 0)
-            throw new ArgumentException("Post listing must contain at least one child", nameof(rawRedditPost));
-
-        var mainPostData = postChildren[0].Data;
-        var bestImage = ExtractBestImage(mainPostData);
-
-        var redditPost = new RedditPost
-        {
-            Post = new RedditPostContent
-            {
-                Id = mainPostData.Id ?? string.Empty,
-                Title = mainPostData.Title ?? throw new InvalidOperationException("Reddit post must have a title"),
-                Author = mainPostData.Author ?? string.Empty,
-                Subreddit = mainPostData.Subreddit ?? string.Empty,
-                Score = mainPostData.Score,
-                Content = mainPostData.Content ?? string.Empty,
-                CreatedUtc = mainPostData.CreatedUtc,
-                ImageUrl = bestImage
-            },
-            Comments = TransformComments(commentsListing)
-        };
-
-        return redditPost;
-    }
-
     private static string? ExtractBestImage(RawRedditCommentData postData)
     {
         // 1. Gallery images (highest priority) - pick the first/largest
@@ -104,22 +69,145 @@
         return extension is ".jpg" or ".jpeg" or ".png" or ".gif" or ".webp" or ".bmp" or ".svg";
     }
 
-    private List<RedditComment> TransformComments(List<RawRedditChild> children)
+    private List<RedditComment> TransformComments(List<RawRedditChild> children, 
+        string subreddit = "", string postId = "")
+    {
+        var comments = new List<RedditComment>();
+
+        foreach (var child in children)
+        {
+            if (child.Kind == "t1")
+            {
+                var commentUrl = $"https://www.reddit.com/r/{subreddit}/comments/{postId}/comment/{child.Data.Id}/";
+                var comment = new RedditComment
+                {
+                    Id = child.Data.Id ?? string.Empty,
+                    Author = child.Data.Author ?? string.Empty,
+                    Score = child.Data.Score,
+                    Content = child.Data.Content ?? string.Empty,
+                    CreatedUtc = child.Data.CreatedUtc,
+                    PostUrl = commentUrl,
+                    Replies = TransformComments(child.Data.Replies, subreddit, postId)
+                };
+
+                comments.Add(comment);
+            }
+        }
+
+        return comments;
+    }
+
+    private List<RedditComment> TransformComments(object? replies, string subreddit = "", string postId = "")
+    {
+        if (replies == null)
+            return [];
+
+        if (replies is string stringReply && stringReply == "")
+            return [];
+
+        if (replies is JsonElement jsonElement)
+        {
+            if (jsonElement.ValueKind == JsonValueKind.Null)
+                return [];
+
+            if (jsonElement.ValueKind == JsonValueKind.String && jsonElement.GetString() == "")
+                return [];
+
+            try
+            {
+                var deserializedListing = JsonSerializer.Deserialize<RawRedditListing>(jsonElement.GetRawText());
+                return TransformComments(deserializedListing, subreddit, postId);
+            }
+            catch
+            {
+                return [];
+            }
+        }
+
+        if (replies is RawRedditListing listing)
+            return TransformComments(listing, subreddit, postId);
+
+        return [];
+    }
+
+    private List<RedditComment> TransformComments(RawRedditListing? replies, string subreddit = "", string postId = "")
+    {
+        if (replies == null)
+            return [];
+
+        if (replies.Data == null)
+            return [];
+
+        if (replies.Data.Children == null)
+            return [];
+
+        return TransformComments(replies.Data.Children, subreddit, postId);
+    }
+
+    public RedditPost Transform(RawRedditPost rawRedditPost)
+    {
+        ArgumentNullException.ThrowIfNull(rawRedditPost);
+        if (rawRedditPost.Count < 2)
+            throw new ArgumentException("Reddit post must have at least 2 listings (post and comments)", nameof(rawRedditPost));
+
+        var postListing = rawRedditPost[0];
+        var commentsListing = rawRedditPost[1];
+
+        var postChildren = postListing.Data?.Children;
+        if (postChildren == null || postChildren.Count == 0)
+            throw new ArgumentException("Post listing must contain at least one child", nameof(rawRedditPost));
+
+        var mainPostData = postChildren[0].Data;
+        var bestImage = ExtractBestImage(mainPostData);
+
+        var subreddit = mainPostData.Subreddit ?? string.Empty;
+        var postId = mainPostData.Id ?? string.Empty;
+        var postUrl = mainPostData.Url ?? string.Empty;
+        var host = "www.reddit.com";
+        if (!string.IsNullOrWhiteSpace(postUrl) && Uri.TryCreate(postUrl, UriKind.Absolute, out var postUri))
+        {
+            host = postUri.Host;
+        }
+
+        var redditPost = new RedditPost
+        {
+            Post = new RedditPostContent
+            {
+                Id = mainPostData.Id ?? string.Empty,
+                Title = mainPostData.Title ?? throw new InvalidOperationException("Reddit post must have a title"),
+                Author = mainPostData.Author ?? string.Empty,
+                Subreddit = subreddit,
+                Score = mainPostData.Score,
+                Content = mainPostData.Content ?? string.Empty,
+                CreatedUtc = mainPostData.CreatedUtc,
+                ImageUrl = bestImage,
+                PostUrl = postUrl
+            },
+            Comments = TransformComments(commentsListing.Data?.Children ?? [], subreddit, postId, host)
+        };
+
+        return redditPost;
+    }
+
+    private List<RedditComment> TransformComments(List<RawRedditChild> children, 
+        string subreddit = "", string postId = "", string host = "www.reddit.com")
     {
         var comments = new List<RedditComment>();
 
         foreach (var child in children)
         {
             if (child.Kind == "t1")
             {
+                var commentUrl = $"https://{host}/r/{subreddit}/comments/{postId}/comment/{child.Data.Id}/";
                 var comment = new RedditComment
                 {
                     Id = child.Data.Id ?? string.Empty,
                     Author = child.Data.Author ?? string.Empty,
                     Score = child.Data.Score,
                     Content = child.Data.Content ?? string.Empty,
                     CreatedUtc = child.Data.CreatedUtc,
-                    Replies = TransformComments(child.Data.Replies)
+                    PostUrl = commentUrl,
+                    Replies = TransformComments(child.Data.Replies, subreddit, postId, host)
                 };
 
                 comments.Add(comment);
@@ -129,7 +217,10 @@
         return comments;
     }
 
-    private List<RedditComment> TransformComments(object? replies)
+    private List<RedditComment> TransformComments(object? replies, 
+        string subreddit = "", 
+        string postId = "", 
+        string host = "www.reddit.com")
     {
         if (replies == null)
             return [];
@@ -148,7 +239,7 @@
             try
             {
                 var deserializedListing = JsonSerializer.Deserialize<RawRedditListing>(jsonElement.GetRawText());
-                return TransformComments(deserializedListing);
+                return TransformComments(deserializedListing, subreddit, postId, host);
             }
             catch
             {
@@ -157,12 +248,16 @@
         }
 
         if (replies is RawRedditListing listing)
-            return TransformComments(listing);
+            return TransformComments(listing, subreddit, postId, host);
 
         return [];
     }
 
-    private List<RedditComment> TransformComments(RawRedditListing? replies)
+    private List<RedditComment> TransformComments(
+        RawRedditListing? replies, 
+        string subreddit = "", 
+        string postId = "", 
+        string host = "www.reddit.com")
     {
         if (replies == null)
             return [];
@@ -173,6 +268,6 @@
         if (replies.Data.Children == null)
             return [];
 
-        return TransformComments(replies.Data.Children);
+        return TransformComments(replies.Data.Children, subreddit, postId, host);
     }
 }

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/Client/RedditPost.cs

@@ -16,6 +16,7 @@ public class RedditPostContent
     public string Content { get; set; } = string.Empty;
     public DateTime CreatedUtc { get; set; }
     public string? ImageUrl { get; set; }
+    public string PostUrl { get; set; } = string.Empty;
 }
 
 public class RedditComment
@@ -26,4 +27,5 @@ public class RedditComment
     public string Content { get; set; } = string.Empty;
     public DateTime CreatedUtc { get; set; }
     public List<RedditComment> Replies { get; set; } = [];
+    public string PostUrl { get; set; } = string.Empty;
 }

src/Elzik.Breef.Infrastructure/ContentExtractors/Reddit/RedditPostContentExtractor.cs

@@ -60,6 +60,7 @@ protected override async Task<UntypedExtract> CreateUntypedExtractAsync(string w
 
         var postId = segments[3];
         var post = await redditPostClient.GetPost(postId);
+        post.Post.PostUrl = webPageUrl;
 
         if (string.IsNullOrWhiteSpace(post.Post.ImageUrl))
         {

src/Elzik.Breef.Infrastructure/SummarisationInstructions/SubredditContent.md

@@ -1,8 +1,31 @@
-You are an expert summarizer. Your task is to summarize the provided text:
- - Summarise text, including HTML entities.
- - Limit summaries to 10% of the original length but never more then 200 words.
- - Ensure accurate attribution of information to the correct entities.
- - Do not include a link to the original articles.
- - Do not include the title in the response.
- - Do not include any metadata in the response.
- - Do not include a code block in the response.
+# Task
+
+Summarize the provided Reddit subreddit JSON data containing posts and nested comments.
+
+## Input Structure
+
+JSON with a "Posts" array, where each post contains:
+- Metadata: Title, Author, Subreddit, Score, Content, CreatedUtc, PostUrl
+- Comments: Array with Author, Score, Content, CreatedUtc, PostUrl and nested Replies
+
+## Requirements
+
+1. **Overview**: Describe the general state/themes of the subreddit
+2. **Posts**: Summarise every post with a thematic summary of its comments
+3. **Summaries**: 
+   - Maximum 200 words OR 10% of original length (whichever is shorter)
+   - For a top-level post, include post title as HTML link to the post URL: `<a href="RedditPostContent.PostUrl">Title</a>`
+   - Where a post's replies are highly scoring, also summarise them and include author attribution with comment links: `<a href="RedditPostContent.PostUrl">@author</a>`. Link to the author's comment, not to the author's profile.
+4. **Exclude**:
+   - Links Subreddit
+   - Root post title
+   - Metadata timestamps/scores
+   - Code blocks
+
+## Output Format
+
+- Strictly well-formatted HTML output
+- Do not include any markdown notation nor put the summary in a codeblock
+- Brief overview of themes covered in this specific JSON document
+- Do not include a general description of the subreddit itself
+- Summaries of the highest scoring top-level posts

tests/Elzik.Breef.Infrastructure.Tests.Integration/ContentExtractors/Reddit/Client/RawRedditPostClientTests.cs

@@ -44,6 +44,7 @@ public async Task GetPost_ValidPostId_ReturnsRedditPost()
             "at companies – 12 weeks\n\nClean code – 6 weeks\n\nApprenticeship at companies – 16 weeks\n\nExam " +
             "thesis – 4 weeks");
         mainPost.Content.ShouldBe(mainPost.SelfText);
+        mainPost.Url.ShouldNotBeNullOrWhiteSpace();
 
         var replies = redditPost[1].Data.Children;
 

tests/Elzik.Breef.Infrastructure.Tests.Integration/ContentExtractors/Reddit/RedditPostContentExtractorTests.cs

@@ -63,6 +63,7 @@ public async Task ExtractAsync_RealRedditPost_ReturnsValidExtract(string url)
         redditPost.Post.ShouldNotBeNull();
         redditPost.Post.Id.ShouldBe("1kqiwzc");
         redditPost.Post.Title.ShouldNotBeNullOrWhiteSpace();
+        redditPost.Post.PostUrl.ShouldNotBeNullOrWhiteSpace();
         redditPost.Comments.ShouldNotBeNull();
     }
 
@@ -136,6 +137,7 @@ public async Task ExtractAsync_ValidPost_ContentContainsCompleteRedditStructure(
         redditPost.Post.Author.ShouldNotBeNullOrEmpty();
         redditPost.Post.Subreddit.ShouldNotBeNullOrEmpty();
         redditPost.Post.CreatedUtc.ShouldNotBe(default);
+        redditPost.Post.PostUrl.ShouldNotBeNullOrWhiteSpace();
         redditPost.Comments.ShouldNotBeNull();
         if (redditPost.Comments.Count != 0)
         {

tests/Elzik.Breef.Infrastructure.Tests.Unit/ContentExtractors/Reddit/Client/RawNewInSubredditTransformerTests.cs

@@ -56,7 +56,8 @@ public async Task Transform_ValidRawNewInSubreddit_ReturnsExpectedStructure()
                 Author = "author1",
                 Score = 100,
                 Content = "Content 1",
-                CreatedUtc = new DateTime(2025, 1, 1, 12, 0, 0, DateTimeKind.Utc)
+                CreatedUtc = new DateTime(2025, 1, 1, 12, 0, 0, DateTimeKind.Utc),
+                PostUrl = "https://reddit.com/r/testsubreddit/comments/post1"
             },
             Comments = []
         };
@@ -70,7 +71,8 @@ public async Task Transform_ValidRawNewInSubreddit_ReturnsExpectedStructure()
                 Author = "author2",
                 Score = 200,
                 Content = "Content 2",
-                CreatedUtc = new DateTime(2025, 1, 1, 13, 0, 0, DateTimeKind.Utc)
+                CreatedUtc = new DateTime(2025, 1, 1, 13, 0, 0, DateTimeKind.Utc),
+                PostUrl = "https://reddit.com/r/testsubreddit/comments/post2"
             },
             Comments = []
         };
@@ -92,13 +94,15 @@ public async Task Transform_ValidRawNewInSubreddit_ReturnsExpectedStructure()
         firstPost.Post.Author.ShouldBe("author1");
         firstPost.Post.Score.ShouldBe(100);
         firstPost.Post.Content.ShouldBe("Content 1");
+        firstPost.Post.PostUrl.ShouldBe("https://reddit.com/r/testsubreddit/comments/post1");
 
         var secondPost = result.Posts[1];
         secondPost.Post.Id.ShouldBe("post2");
         secondPost.Post.Title.ShouldBe("Test Post 2");
         secondPost.Post.Author.ShouldBe("author2");
         secondPost.Post.Score.ShouldBe(200);
         secondPost.Post.Content.ShouldBe("Content 2");
+        secondPost.Post.PostUrl.ShouldBe("https://reddit.com/r/testsubreddit/comments/post2");
     }
 
     [Fact]
@@ -319,7 +323,8 @@ public async Task Transform_SinglePost_ReturnsNewInSubredditWithOnePost()
                 Score = 42,
                 Subreddit = "test",
                 CreatedUtc = new DateTime(2025, 1, 1, 14, 0, 0, DateTimeKind.Utc),
-                ImageUrl = "https://example.com/image.jpg"
+                ImageUrl = "https://example.com/image.jpg",
+                PostUrl = "https://reddit.com/r/test/single_post"
             },
             Comments =
             [
@@ -352,6 +357,7 @@ public async Task Transform_SinglePost_ReturnsNewInSubredditWithOnePost()
         post.Post.Score.ShouldBe(42);
         post.Post.Subreddit.ShouldBe("test");
         post.Post.ImageUrl.ShouldBe("https://example.com/image.jpg");
+        post.Post.PostUrl.ShouldNotBeNullOrEmpty();
         post.Comments.Count.ShouldBe(1);
         post.Comments[0].Content.ShouldBe("Great post!");
     }