Encapsulate helpers constants and functions inside it's own Helpers module

src/invidious/channels/community.cr

@@ -127,7 +127,7 @@ def extract_channel_community(items, *, ucid, locale, format, thin_mode, is_sing
 
               reply_count = short_text_to_number(post.dig?("actionButtons", "commentActionButtonsRenderer", "replyButton", "buttonRenderer", "text", "simpleText").try &.as_s || "0")
 
-              json.field "content", html_to_content(content_html)
+              json.field "content", Helpers.html_to_content(content_html)
               json.field "contentHtml", content_html
 
               json.field "published", published.to_unix

src/invidious/comments/youtube.cr

@@ -254,7 +254,7 @@ module Invidious::Comments
                 end
 
                 content_html = html_content || ""
-                json.field "content", html_to_content(content_html)
+                json.field "content", Helpers.html_to_content(content_html)
                 json.field "contentHtml", content_html
 
                 if published_text != nil

src/invidious/helpers/helpers.cr

@@ -1,7 +1,5 @@
 require "./macros"
 
-TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
-
 struct Nonce
   include DB::Serializable
 
@@ -24,60 +22,124 @@ struct Annotation
   property annotations : String
 end
 
-def html_to_content(description_html : String)
-  description = description_html.gsub(/(<br>)|(<br\/>)/, {
-    "<br>":  "\n",
-    "<br/>": "\n",
-  })
+module Helpers
+  extend self
 
-  if !description.empty?
-    description = XML.parse_html(description).content.strip("\n ")
-  end
+  private TEST_IDS = {"AgbeGFYluEA", "BaW_jenozKc", "a9LDPn-MO4I", "ddFvjfvPnqk", "iqKdEhx-dD4"}
 
-  return description
-end
+  def html_to_content(description_html : String)
+    description = description_html.gsub(/(<br>)|(<br\/>)/, {
+      "<br>":  "\n",
+      "<br/>": "\n",
+    })
+
+    if !description.empty?
+      description = XML.parse_html(description).content.strip("\n ")
+    end
 
-def cache_annotation(id, annotations)
-  if !CONFIG.cache_annotations
-    return
+    return description
   end
 
-  body = XML.parse(annotations)
-  nodeset = body.xpath_nodes(%q(/document/annotations/annotation))
+  def cache_annotation(id, annotations)
+    if !CONFIG.cache_annotations
+      return
+    end
+
+    body = XML.parse(annotations)
+    nodeset = body.xpath_nodes(%q(/document/annotations/annotation))
 
-  return if nodeset == 0
+    return if nodeset == 0
 
-  has_legacy_annotations = false
-  nodeset.each do |node|
-    if !{"branding", "card", "drawer"}.includes? node["type"]?
-      has_legacy_annotations = true
-      break
+    has_legacy_annotations = false
+    nodeset.each do |node|
+      if !{"branding", "card", "drawer"}.includes? node["type"]?
+        has_legacy_annotations = true
+        break
+      end
     end
+
+    Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations
   end
 
-  Invidious::Database::Annotations.insert(id, annotations) if has_legacy_annotations
-end
+  def create_notification_stream(env, topics, connection_channel)
+    connection = Channel(PQ::Notification).new(8)
+    connection_channel.send({true, connection})
+
+    locale = env.get("preferences").as(Preferences).locale
+
+    since = env.params.query["since"]?.try &.to_i?
+    id = 0
+
+    if topics.includes? "debug"
+      spawn do
+        begin
+          loop do
+            time_span = [0, 0, 0, 0]
+            time_span[rand(4)] = rand(30) + 5
+            published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
+            video_id = TEST_IDS[rand(TEST_IDS.size)]
+
+            video = get_video(video_id)
+            video.published = published
+            response = JSON.parse(video.to_json(locale, nil))
+
+            env.response.puts "id: #{id}"
+            env.response.puts "data: #{response.to_json}"
+            env.response.puts
+            env.response.flush
 
-def create_notification_stream(env, topics, connection_channel)
-  connection = Channel(PQ::Notification).new(8)
-  connection_channel.send({true, connection})
+            id += 1
 
-  locale = env.get("preferences").as(Preferences).locale
+            sleep 1.minute
+            Fiber.yield
+          end
+        rescue ex
+        end
+      end
+    end
 
-  since = env.params.query["since"]?.try &.to_i?
-  id = 0
+    spawn do
+      begin
+        if since
+          since_unix = Time.unix(since.not_nil!)
+
+          topics.try &.each do |topic|
+            case topic
+            when .match(/UC[A-Za-z0-9_-]{22}/)
+              Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
+                response = JSON.parse(video.to_json(locale))
+
+                env.response.puts "id: #{id}"
+                env.response.puts "data: #{response.to_json}"
+                env.response.puts
+                env.response.flush
+
+                id += 1
+              end
+            else
+              # TODO
+            end
+          end
+        end
+      end
+    end
 
-  if topics.includes? "debug"
     spawn do
       begin
         loop do
-          time_span = [0, 0, 0, 0]
-          time_span[rand(4)] = rand(30) + 5
-          published = Time.utc - Time::Span.new(days: time_span[0], hours: time_span[1], minutes: time_span[2], seconds: time_span[3])
-          video_id = TEST_IDS[rand(TEST_IDS.size)]
+          event = connection.receive
+
+          notification = JSON.parse(event.payload)
+          topic = notification["topic"].as_s
+          video_id = notification["videoId"].as_s
+          published = notification["published"].as_i64
+
+          if !topics.try &.includes? topic
+            next
+          end
 
           video = get_video(video_id)
-          video.published = published
+          video.published = Time.unix(published)
           response = JSON.parse(video.to_json(locale, nil))
 
           env.response.puts "id: #{id}"
@@ -86,117 +148,59 @@ def create_notification_stream(env, topics, connection_channel)
           env.response.flush
 
           id += 1
-
-          sleep 1.minute
-          Fiber.yield
         end
       rescue ex
+      ensure
+        connection_channel.send({false, connection})
       end
     end
-  end
 
-  spawn do
-    begin
-      if since
-        since_unix = Time.unix(since.not_nil!)
-
-        topics.try &.each do |topic|
-          case topic
-          when .match(/UC[A-Za-z0-9_-]{22}/)
-            Invidious::Database::ChannelVideos.select_notfications(topic, since_unix).each do |video|
-              response = JSON.parse(video.to_json(locale))
-
-              env.response.puts "id: #{id}"
-              env.response.puts "data: #{response.to_json}"
-              env.response.puts
-              env.response.flush
-
-              id += 1
-            end
-          else
-            # TODO
-          end
-        end
-      end
-    end
-  end
-
-  spawn do
     begin
+      # Send heartbeat
       loop do
-        event = connection.receive
-
-        notification = JSON.parse(event.payload)
-        topic = notification["topic"].as_s
-        video_id = notification["videoId"].as_s
-        published = notification["published"].as_i64
-
-        if !topics.try &.includes? topic
-          next
-        end
-
-        video = get_video(video_id)
-        video.published = Time.unix(published)
-        response = JSON.parse(video.to_json(locale, nil))
-
-        env.response.puts "id: #{id}"
-        env.response.puts "data: #{response.to_json}"
+        env.response.puts ":keepalive #{Time.utc.to_unix}"
         env.response.puts
         env.response.flush
-
-        id += 1
+        sleep (20 + rand(11)).seconds
       end
     rescue ex
     ensure
       connection_channel.send({false, connection})
     end
   end
 
-  begin
-    # Send heartbeat
-    loop do
-      env.response.puts ":keepalive #{Time.utc.to_unix}"
-      env.response.puts
-      env.response.flush
-      sleep (20 + rand(11)).seconds
-    end
-  rescue ex
-  ensure
-    connection_channel.send({false, connection})
+  def extract_initial_data(body) : Hash(String, JSON::Any)
+    return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>{.*?});<\/script>/mx).try &.["info"] || "{}").as_h
   end
-end
-
-def extract_initial_data(body) : Hash(String, JSON::Any)
-  return JSON.parse(body.match(/(window\["ytInitialData"\]|var\s*ytInitialData)\s*=\s*(?<info>{.*?});<\/script>/mx).try &.["info"] || "{}").as_h
-end
 
-def proxy_file(response, env)
-  if response.headers.includes_word?("Content-Encoding", "gzip")
-    Compress::Gzip::Writer.open(env.response) do |deflate|
-      IO.copy response.body_io, deflate
-    end
-  elsif response.headers.includes_word?("Content-Encoding", "deflate")
-    Compress::Deflate::Writer.open(env.response) do |deflate|
-      IO.copy response.body_io, deflate
+  def proxy_file(response, env)
+    if response.headers.includes_word?("Content-Encoding", "gzip")
+      Compress::Gzip::Writer.open(env.response) do |deflate|
+        IO.copy response.body_io, deflate
+      end
+    elsif response.headers.includes_word?("Content-Encoding", "deflate")
+      Compress::Deflate::Writer.open(env.response) do |deflate|
+        IO.copy response.body_io, deflate
+      end
+    else
+      IO.copy response.body_io, env.response
     end
-  else
-    IO.copy response.body_io, env.response
   end
-end
 
-# Fetch the playback requests tracker from the statistics endpoint.
-#
-# Creates a new tracker when unavailable.
-def get_playback_statistic
-  if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
-    tracker = {
-      "totalRequests"      => 0_i64,
-      "successfulRequests" => 0_i64,
-      "ratio"              => 0_f64,
-    }
-
-    Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
-  end
+  # Fetch the playback requests tracker from the statistics endpoint.
+  #
+  # Creates a new tracker when unavailable.
+  def get_playback_statistic
+    if (tracker = Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"]) && tracker.as(Hash).empty?
+      tracker = {
+        "totalRequests"      => 0_i64,
+        "successfulRequests" => 0_i64,
+        "ratio"              => 0_f64,
+      }
+
+      Invidious::Jobs::StatisticsRefreshJob::STATISTICS["playback"] = tracker
+    end
 
-  return tracker.as(Hash(String, Int64 | Float64))
+    return tracker.as(Hash(String, Int64 | Float64))
+  end
 end

src/invidious/helpers/serialized_yt_data.cr

@@ -53,7 +53,7 @@ struct SearchVideo
             xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
           end
 
-          xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
+          xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text Helpers.html_to_content(self.description_html) }
         end
       end
 
@@ -63,7 +63,7 @@ struct SearchVideo
         xml.element("media:title") { xml.text self.title }
         xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
           width: "320", height: "180")
-        xml.element("media:description") { xml.text html_to_content(self.description_html) }
+        xml.element("media:description") { xml.text Helpers.html_to_content(self.description_html) }
       end
 
       xml.element("media:community") do
@@ -111,7 +111,7 @@ struct SearchVideo
         Invidious::JSONify::APIv1.thumbnails(json, self.id)
       end
 
-      json.field "description", html_to_content(self.description_html)
+      json.field "description", Helpers.html_to_content(self.description_html)
       json.field "descriptionHtml", self.description_html
 
       json.field "viewCount", self.views
@@ -255,7 +255,7 @@ struct SearchChannel
       json.field "videoCount", self.video_count
       json.field "channelHandle", self.channel_handle
 
-      json.field "description", html_to_content(self.description_html)
+      json.field "description", Helpers.html_to_content(self.description_html)
       json.field "descriptionHtml", self.description_html
     end
   end

src/invidious/mixes.cr

@@ -27,7 +27,7 @@ def fetch_mix(rdid, video_id, cookies = nil, locale = nil)
 
   video_id = "CvFH_6DNRCY" if rdid.starts_with? "OLAK5uy_"
   response = YT_POOL.client &.get("/watch?v=#{video_id}&list=#{rdid}&gl=US&hl=en", headers)
-  initial_data = extract_initial_data(response.body)
+  initial_data = Helpers.extract_initial_data(response.body)
 
   if !initial_data["contents"]["twoColumnWatchNextResults"]["playlist"]?
     raise InfoException.new("Could not create mix.")

src/invidious/playlists.cr

@@ -199,7 +199,7 @@ struct InvidiousPlaylist
       json.field "authorUrl", nil
       json.field "authorThumbnails", [] of String
 
-      json.field "description", html_to_content(self.description_html)
+      json.field "description", Helpers.html_to_content(self.description_html)
       json.field "descriptionHtml", self.description_html
       json.field "videoCount", self.video_count