Use with-utf8 to fix text encoding problems

Author: rnons

.circleci/config.yml

@@ -36,6 +36,7 @@ jobs:
           working_directory: backend
           command: |
             source ~/.profile
+            cachix use rnons
             nix-build -j2 --cores 2
             cp -r result/bin bin
 

backend/default.nix

@@ -2,4 +2,7 @@
 , compiler ? "ghc864"
 }:
 
-nixpkgs.pkgs.haskell.packages.${compiler}.callPackage ./ted2srt.nix {}
+let
+  overrides = import ../nix/all.nix { inherit nixpkgs compiler; };
+in
+nixpkgs.pkgs.haskell.packages.${compiler}.callPackage ./ted2srt.nix overrides

backend/package.yaml

@@ -42,6 +42,7 @@ dependencies:
 - unordered-containers
 - vector
 - wai
+- with-utf8
 - xml-conduit
 
 # The library contains all of our application code. The executable

backend/src/Web/TED.hs

@@ -19,6 +19,7 @@ import           RIO
 import           RIO.List.Partial     (head)
 import           System.Directory
 import           System.IO            (hPutStr, hPutStrLn, openFile, print)
+import qualified System.IO.Utf8 as Utf8
 import           Text.Printf
 
 import           Web.TED.API          as TED
@@ -141,7 +142,7 @@ oneSub sub = do
                   "%02d:%02d:%02d,%03d\n%s\n\n"
              else "%d\n%02d:%02d:%02d.%03d --> " ++
                   "%02d:%02d:%02d.%03d\n%s\n\n"
-    ppr h (c,i) = do
+    ppr h (c,i) = Utf8.withHandle h $ do
         let st = startTime c + floor (timeLag sub)
             sh = st `div` 1000 `div` 3600
             sm = st `div` 1000 `mod` 3600 `div` 60
@@ -165,9 +166,10 @@ oneTxt sub = do
       else do
           txt <- TED.getTalkTranscript (talkId sub) (head $ language sub)
           -- Prepend the UTF-8 byte order mark to do Windows user a favor.
-          withBinaryFile path WriteMode $ \h ->
+          withBinaryFile path WriteMode $ \h -> Utf8.withHandle h $
               hPutStr h "\xef\xbb\xbf"
-          T.appendFile path txt
+          Utf8.withFile path AppendMode $ \h ->
+            T.hPutStrLn h txt
           return $ Just path
 
 oneLrc :: Subtitle -> IO (Maybe FilePath)

backend/stack.yaml

@@ -41,6 +41,7 @@ packages:
 # (e.g., acme-missiles-0.3)
 extra-deps:
 - hedis-0.12.9
+- with-utf8-1.0.0.0
 
 # Override default flag values for local packages and extra-deps
 flags: {}

backend/ted2srt.nix

@@ -5,7 +5,8 @@
 , persistent, persistent-postgresql, persistent-template
 , raw-strings-qq, regex-posix, rio, servant-lucid, servant-server
 , stdenv, system-filepath, text, time, transformers
-, unordered-containers, vector, wai, wai-extra, warp, xml-conduit
+, unordered-containers, vector, wai, wai-extra, warp, with-utf8
+, xml-conduit
 , postgresql_11}:
 mkDerivation {
   pname = "ted2srt";
@@ -20,7 +21,7 @@ mkDerivation {
     monad-logger mtl network persistent persistent-postgresql
     persistent-template raw-strings-qq regex-posix rio servant-lucid
     servant-server system-filepath text time transformers
-    unordered-containers vector wai xml-conduit
+    unordered-containers vector wai with-utf8 xml-conduit
   ];
   librarySystemDepends = [ postgresql_11 ];
   libraryToolDepends = [ hpack ];
@@ -31,7 +32,8 @@ mkDerivation {
     monad-logger mtl network persistent persistent-postgresql
     persistent-template raw-strings-qq regex-posix rio servant-lucid
     servant-server system-filepath text time transformers
-    unordered-containers vector wai wai-extra warp xml-conduit
+    unordered-containers vector wai wai-extra warp with-utf8
+    xml-conduit
   ];
   doHaddock = false;
   prePatch = "hpack";

nix/all.nix

@@ -0,0 +1,10 @@
+{ nixpkgs ? import ./nixpkgs.nix {}
+, compiler ? "ghc864"
+}:
+
+let
+  global = { inherit nixpkgs compiler; };
+
+in {
+  with-utf8 = import ./pkgs/with-utf8.nix global;
+}

nix/pkgs/with-utf8.nix

@@ -0,0 +1,24 @@
+{ nixpkgs
+, compiler
+, callPackage ? nixpkgs.pkgs.haskell.packages.${compiler}.callPackage
+}:
+let with-utf8 = callPackage
+  ({ mkDerivation, base, deepseq, hedgehog, HUnit, safe-exceptions
+    , tasty, tasty-discover, tasty-hedgehog, tasty-hunit, temporary
+    , text, unix, stdenv
+    }:
+    mkDerivation {
+      pname = "with-utf8";
+      version = "1.0.0.0";
+      sha256 = "06xznaszw7d6rznvzhzw3y4z31b4vx4djms85rq4qsbpfbdrh2zc";
+      libraryHaskellDepends = [ base safe-exceptions text ];
+      testHaskellDepends = [
+        base deepseq hedgehog HUnit safe-exceptions tasty tasty-hedgehog
+        tasty-hunit temporary text unix
+      ];
+      testToolDepends = [ tasty-discover ];
+      description = "Get your IO right on the first try";
+      license = stdenv.lib.licenses.mpl20;
+    }) {};
+in nixpkgs.pkgs.haskell.lib.dontHaddock
+  (nixpkgs.pkgs.haskell.lib.dontCheck with-utf8)