bb.edn

{:tasks
 {:requires ([clojure.string :as str]
             [babashka.fs :as fs]
             [babashka.curl :as curl])

  haata
  {:task
   (let [target-dir "resources/haata"
         build-url (fn [i] (str "https://chart-studio.plotly.com/~haata/" i ".csv"))
         save-csv! (fn [url res]
                     (let [[_ title]
                           (re-find (re-pattern "<title>(.*)</title>")
                                    (:body (curl/get (str/replace url (re-pattern ".csv$") ""))))
                           [switch-name] (str/split title (re-pattern " \\| "))
                           file-name (str switch-name ".csv")]
                       (when-not (or (str/blank? title)
                                     (str/includes? (str/lower-case title) "untitled")
                                     (str/blank? switch-name))
                         (println (str "Saving file " file-name))
                         (spit (fs/file target-dir file-name) (:body res)))))]
     (when (fs/exists? target-dir)
       (println "Clearing old CSV files")
       (fs/delete-tree target-dir))
     (fs/create-dir target-dir)
     (dotimes [i 1000]
       (let [url (build-url i)
             _ (println (str "Checking " url))
             res (curl/get url {:throw false})]
         (when (and (= 200 (:status res))
                    (= "text/csv" (get-in res [:headers "content-type"])))
           (println "Found CSV")
           (save-csv! url res)))))}}}