diff --git a/src/invidious/channels/community.cr b/src/invidious/channels/community.cr index 4256230cb..6884602bb 100644 --- a/src/invidious/channels/community.cr +++ b/src/invidious/channels/community.cr @@ -127,7 +127,7 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0") - json.field "content", html_to_content(content_html) + json.field "content", Helpers.html_to_content(content_html) json.field "contentHtml", content_html json.field "published", published.to_unix diff --git a/src/invidious/comments/youtube.cr b/src/invidious/comments/youtube.cr index e923b2f8d..2f8f2932c 100644 --- a/src/invidious/comments/youtube.cr +++ b/src/invidious/comments/youtube.cr @@ -254,7 +254,7 @@ module Invidious::Comments end content_html = html_content || "" - json.field "content", html_to_content(content_html) + json.field "content", Helpers.html_to_content(content_html) json.field "contentHtml", content_html if published_text != nil diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index ab694b1f1..a08135eaa 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -1,7 +1,5 @@ require "./macros" -TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} - struct Nonce include DB::Serializable @@ -24,60 +22,124 @@ struct Annotation property annotations : String end -def html_to_content(description_html : String) - description = description_html.gsub(/(
)|()/, { - "
": "\n", - "
": "\n", - }) +module Helpers + extend self - if !description.empty? - description = XML.parse_html(description).content.strip("\n ") - end + private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"} - return description -end + def html_to_content(description_html : String) + description = description_html.gsub(/(
)|()/, { + "
": "\n", + "
": "\n", + }) + + if !description.empty? + description = XML.parse_html(description).content.strip("\n ") + end -def cache_annotation(id, annotations) - if !CONFIG.cache_annotations - return + return description end - body = XML.parse(annotations) - nodeset = body.xpath_nodes(%q(/document/annotations/annotation)) + def cache_annotation(id, annotations) + if !CONFIG.cache_annotations + return + end + + body = XML.parse(annotations) + nodeset = body.xpath_nodes(%q(/document/annotations/annotation)) - return if nodeset == 0 + return if nodeset == 0 - has_legacy_annotations = false - nodeset.each do |node| - if !{"branding", "card", "drawer"}.includes? node["type"]? - has_legacy_annotations = true - break + has_legacy_annotations = false + nodeset.each do |node| + if !{"branding", "card", "drawer"}.includes? node["type"]? + has_legacy_annotations = true + break + end end + + Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations end - Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations -end + def create_notification_stream(env, topics, connection_channel) + connection = Channel(PQ::Notification).new(8) + connection_channel.send({true, connection}) + + locale = env.get("preferences").as(Preferences).locale + + since = env.params.query["since"]?.try &.to_i? + id = 0 + + if topics.includes? "debug" + spawn do + begin + loop do + time_span = [0, 0, 0, 0] + time_span[rand(4)] = rand(30) + 5 + published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) + video_id = TEST_IDS[rand(TEST_IDS.size)] + + video = get_video(video_id) + video.published = published + response = JSON.parse(video.to_json(locale, nil)) + + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush -def create_notification_stream(env, topics, connection_channel) - connection = Channel(PQ::Notification).new(8) - connection_channel.send({true, connection}) + id += 1 - locale = env.get("preferences").as(Preferences).locale + sleep 1.minute + Fiber.yield + end + rescue ex + end + end + end - since = env.params.query["since"]?.try &.to_i? - id = 0 + spawn do + begin + if since + since_unix = Time.unix(since.not_nil!) + + topics.try &.each do |topic| + case topic + when .match(/UC[A-Za-z0-9_-]{22}/) + Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| + response = JSON.parse(video.to_json(locale)) + + env.response.puts "id: #{id}" + env.response.puts "data: #{response.to_json}" + env.response.puts + env.response.flush + + id += 1 + end + else + # TODO + end + end + end + end + end - if topics.includes? "debug" spawn do begin loop do - time_span = [0, 0, 0, 0] - time_span[rand(4)] = rand(30) + 5 - published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3]) - video_id = TEST_IDS[rand(TEST_IDS.size)] + event = connection.receive + + notification = JSON.parse(event.payload) + topic = notification["topic"].as_s + video_id = notification["videoId"].as_s + published = notification["published"].as_i64 + + if !topics.try &.includes? topic + next + end video = get_video(video_id) - video.published = published + video.published = Time.unix(published) response = JSON.parse(video.to_json(locale, nil)) env.response.puts "id: #{id}" @@ -86,65 +148,20 @@ def create_notification_stream(env, topics, connection_channel) env.response.flush id += 1 - - sleep 1.minute - Fiber.yield end rescue ex + ensure + connection_channel.send({false, connection}) end end - end - spawn do - begin - if since - since_unix = Time.unix(since.not_nil!) - - topics.try &.each do |topic| - case topic - when .match(/UC[A-Za-z0-9_-]{22}/) - Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video| - response = JSON.parse(video.to_json(locale)) - - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" - env.response.puts - env.response.flush - - id += 1 - end - else - # TODO - end - end - end - end - end - - spawn do begin + # Send heartbeat loop do - event = connection.receive - - notification = JSON.parse(event.payload) - topic = notification["topic"].as_s - video_id = notification["videoId"].as_s - published = notification["published"].as_i64 - - if !topics.try &.includes? topic - next - end - - video = get_video(video_id) - video.published = Time.unix(published) - response = JSON.parse(video.to_json(locale, nil)) - - env.response.puts "id: #{id}" - env.response.puts "data: #{response.to_json}" + env.response.puts ":keepalive #{Time.utc.to_unix}" env.response.puts env.response.flush - - id += 1 + sleep (20 + rand(11)).seconds end rescue ex ensure @@ -152,51 +169,38 @@ def create_notification_stream(env, topics, connection_channel) end end - begin - # Send heartbeat - loop do - env.response.puts ":keepalive #{Time.utc.to_unix}" - env.response.puts - env.response.flush - sleep (20 + rand(11)).seconds - end - rescue ex - ensure - connection_channel.send({false, connection}) + def extract_initial_data(body) : Hash(String, JSON::Any) + return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?{.*?});<\/script>/mx).try &.["info"] || "{}").as_h end -end - -def extract_initial_data(body) : Hash(String, JSON::Any) - return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?{.*?});<\/script>/mx).try &.["info"] || "{}").as_h -end -def proxy_file(response, env) - if response.headers.includes_word?("Content-Encoding", "gzip") - Compress::Gzip::Writer.open(env.response) do |deflate| - IO.copy response.body_io, deflate - end - elsif response.headers.includes_word?("Content-Encoding", "deflate") - Compress::Deflate::Writer.open(env.response) do |deflate| - IO.copy response.body_io, deflate + def proxy_file(response, env) + if response.headers.includes_word?("Content-Encoding", "gzip") + Compress::Gzip::Writer.open(env.response) do |deflate| + IO.copy response.body_io, deflate + end + elsif response.headers.includes_word?("Content-Encoding", "deflate") + Compress::Deflate::Writer.open(env.response) do |deflate| + IO.copy response.body_io, deflate + end + else + IO.copy response.body_io, env.response end - else - IO.copy response.body_io, env.response end -end -# Fetch the playback requests tracker from the statistics endpoint. -# -# Creates a new tracker when unavailable. -def get_playback_statistic - if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? - tracker = { - "totalRequests" => 0_i64, - "successfulRequests" => 0_i64, - "ratio" => 0_f64, - } - - Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker - end + # Fetch the playback requests tracker from the statistics endpoint. + # + # Creates a new tracker when unavailable. + def get_playback_statistic + if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty? + tracker = { + "totalRequests" => 0_i64, + "successfulRequests" => 0_i64, + "ratio" => 0_f64, + } + + Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker + end - return tracker.as(Hash(String, Int64 | Float64)) + return tracker.as(Hash(String, Int64 | Float64)) + end end diff --git a/src/invidious/helpers/serialized_yt_data.cr b/src/invidious/helpers/serialized_yt_data.cr index 2796a8dc6..1e73f796c 100644 --- a/src/invidious/helpers/serialized_yt_data.cr +++ b/src/invidious/helpers/serialized_yt_data.cr @@ -53,7 +53,7 @@ struct SearchVideo xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg") end - xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) } + xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) } end end @@ -63,7 +63,7 @@ struct SearchVideo xml.element("media:title") { xml.text self.title } xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg", width: "320", height: "180") - xml.element("media:description") { xml.text html_to_content(self.description_html) } + xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) } end xml.element("media:community") do @@ -111,7 +111,7 @@ struct SearchVideo Invidious::JSONify::APIv1.thumbnails(json, self.id) end - json.field "description", html_to_content(self.description_html) + json.field "description", Helpers.html_to_content(self.description_html) json.field "descriptionHtml", self.description_html json.field "viewCount", self.views @@ -255,7 +255,7 @@ struct SearchChannel json.field "videoCount", self.video_count json.field "channelHandle", self.channel_handle - json.field "description", html_to_content(self.description_html) + json.field "description", Helpers.html_to_content(self.description_html) json.field "descriptionHtml", self.description_html end end diff --git a/src/invidious/mixes.cr b/src/invidious/mixes.cr index 28ff0ff6e..7dd8a8309 100644 --- a/src/invidious/mixes.cr +++ b/src/invidious/mixes.cr @@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil) video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_" response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers) - initial_data = extract_initial_data(response.body) + initial_data = Helpers.extract_initial_data(response.body) if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]? raise InfoException.new("Could not create mix.") diff --git a/src/invidious/playlists.cr b/src/invidious/playlists.cr index eb084331b..60fab63ca 100644 --- a/src/invidious/playlists.cr +++ b/src/invidious/playlists.cr @@ -199,7 +199,7 @@ struct InvidiousPlaylist json.field "authorUrl", nil json.field "authorThumbnails", [] of String - json.field "description", html_to_content(self.description_html) + json.field "description", Helpers.html_to_content(self.description_html) json.field "descriptionHtml", self.description_html json.field "videoCount", self.video_count diff --git a/src/invidious/routes/api/v1/authenticated.cr b/src/invidious/routes/api/v1/authenticated.cr index a35d2f2b2..7021321f8 100644 --- a/src/invidious/routes/api/v1/authenticated.cr +++ b/src/invidious/routes/api/v1/authenticated.cr @@ -8,7 +8,7 @@ module Invidious::Routes::API::V1::Authenticated # topics = env.params.body["topics"]?.try &.split(",").uniq.first(1000) # topics ||= [] of String - # create_notification_stream(env, topics, connection_channel) + # Helpers.create_notification_stream(env, topics, connection_channel) # end def self.get_preferences(env) @@ -485,6 +485,6 @@ module Invidious::Routes::API::V1::Authenticated topics = raw_topics.try &.split(",").uniq.first(1000) topics ||= [] of String - create_notification_stream(env, topics, CONNECTION_CHANNEL) + Helpers.create_notification_stream(env, topics, CONNECTION_CHANNEL) end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index f8060342c..b0545bdd5 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -97,7 +97,7 @@ module Invidious::Routes::API::V1::Channels json.field "autoGenerated", channel.auto_generated json.field "ageGated", channel.is_age_gated json.field "isFamilyFriendly", channel.is_family_friendly - json.field "description", html_to_content(channel.description_html) + json.field "description", Helpers.html_to_content(channel.description_html) json.field "descriptionHtml", channel.description_html json.field "allowedRegions", channel.allowed_regions diff --git a/src/invidious/routes/api/v1/videos.cr b/src/invidious/routes/api/v1/videos.cr index fc3de6957..5f98321ba 100644 --- a/src/invidious/routes/api/v1/videos.cr +++ b/src/invidious/routes/api/v1/videos.cr @@ -300,7 +300,7 @@ module Invidious::Routes::API::V1::Videos annotations = response.body - cache_annotation(id, annotations) + Helpers.cache_annotation(id, annotations) end else # "youtube" response = YT_POOL.client &.get("/annotations_invideo?video_id=#{id}") diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 51d85dfec..c06955c03 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -96,7 +96,7 @@ module Invidious::Routes::Images break end - proxy_file(response, env) + Helpers.proxy_file(response, env) end rescue ex end @@ -148,6 +148,6 @@ module Invidious::Routes::Images return env.response.headers.delete("Transfer-Encoding") end - return proxy_file(response, env) + return Helpers.proxy_file(response, env) end end diff --git a/src/invidious/routes/video_playback.cr b/src/invidious/routes/video_playback.cr index 7c01aa36e..f16a5a58d 100644 --- a/src/invidious/routes/video_playback.cr +++ b/src/invidious/routes/video_playback.cr @@ -83,7 +83,7 @@ module Invidious::Routes::VideoPlayback # Remove the Range header added previously. headers.delete("Range") if range_header.nil? - playback_statistics = get_playback_statistic() + playback_statistics = Helpers.get_playback_statistic playback_statistics["totalRequests"] += 1 if response.status_code >= 400 @@ -195,7 +195,7 @@ module Invidious::Routes::VideoPlayback end end - proxy_file(resp, env) + Helpers.proxy_file(resp, env) end rescue ex if ex.message != "Error reading socket: Connection reset by peer" diff --git a/src/invidious/search/processors.cr b/src/invidious/search/processors.cr index 25edb9362..ecf9aeb31 100644 --- a/src/invidious/search/processors.cr +++ b/src/invidious/search/processors.cr @@ -21,7 +21,7 @@ module Invidious::Search if response.status_code == 404 response = YT_POOL.client &.get("/user/#{query.channel}") response = YT_POOL.client &.get("/c/#{query.channel}") if response.status_code == 404 - initial_data = extract_initial_data(response.body) + initial_data = Helpers.extract_initial_data(response.body) ucid = initial_data.dig?("header", "c4TabbedHeaderRenderer", "channelId").try(&.as_s?) raise ChannelSearchException.new(query.channel) if !ucid else diff --git a/src/invidious/user/exports.cr b/src/invidious/user/exports.cr index b52503c93..3fa468bad 100644 --- a/src/invidious/user/exports.cr +++ b/src/invidious/user/exports.cr @@ -15,7 +15,7 @@ struct Invidious::User playlists.each do |playlist| json.object do json.field "title", playlist.title - json.field "description", html_to_content(playlist.description_html) + json.field "description", Helpers.html_to_content(playlist.description_html) json.field "privacy", playlist.privacy.to_s json.field "videos" do json.array do diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 8114ad684..d9107d5ac 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -84,7 +84,7 @@ def extract_video_info(video_id : String) # Although technically not a call to /videoplayback the fact that YouTube is returning the # wrong video means that we should count it as a failure. - get_playback_statistic()["totalRequests"] += 1 + Helpers.get_playback_statistic["totalRequests"] += 1 return { "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),