Use openssl

berlin-scraper/app/Main.hs

@@ -7,7 +7,7 @@
 
 module Main where
 
-import Control.Lens (view)
+import Control.Lens (view, (&), (.~))
 import Control.Monad (when)
 import Control.Monad.IO.Class
 import Data.Aeson hiding (Options)
@@ -19,20 +19,25 @@ import Data.Text.Encoding qualified as Text
 import Data.Text.IO qualified as Text
 import Database.SQLite.Simple
 import GHC.Generics
+import Network.HTTP.Client.OpenSSL
 import Network.Wreq hiding (Options)
 import Options.Generic
+import Servant.Client (mkClientEnv)
 import System.Exit (exitFailure)
 import Telegram.Bot.API
 import Telegram.Bot.Simple
 import Telegram.Bot.Simple.Debug
 import Text.HTML.Scalpel
+import qualified Network.Wreq as Wreq
 
 main :: IO ()
-main = do
+main = withOpenSSL $ do
   opts <- unwrapRecord "Berlin Scrapper"
   token <- getEnvToken "TELEGRAM_BOT_TOKEN"
-  clientEnv <- defaultTelegramClientEnv token
-  startBot (traceBotDefault $ botApp opts) clientEnv >>= \case
+  httpMgr <- newOpenSSLManager
+  let clientEnv = mkClientEnv httpMgr (botBaseUrl token)
+      wreqOpts = Wreq.defaults & Wreq.manager .~ Right httpMgr
+  startBot (traceBotDefault $ botApp opts wreqOpts) clientEnv >>= \case
     Left err -> do
       putStrLn $ "Bot failed with: " <> show err
       exitFailure
@@ -70,8 +75,8 @@ instance FromJSON IBWResponse
 
 -- * Parsing
 
-queryIBW :: IO IBWResponse
-queryIBW = do
+queryIBW :: Wreq.Options -> IO IBWResponse
+queryIBW wreqOpts = do
   let reqBody =
         [ partText "q" "wf-save-srch",
           partText "save" "false",
@@ -107,7 +112,7 @@ queryIBW = do
           partText "bez[]" "11_00"
         ]
   let link = "https://inberlinwohnen.de/wp-content/themes/ibw/skript/wohnungsfinder.php"
-  resBS <- view responseBody <$> post link reqBody
+  resBS <- view responseBody <$> postWith wreqOpts link reqBody
   pure $ fromJust $ decode @IBWResponse resBS
 
 scrapeOffers :: Scraper Text [Offer]
@@ -169,13 +174,13 @@ type Model = ()
 newtype Action = StartChat Chat
   deriving (Show)
 
-botApp :: Options Unwrapped -> BotApp Model Action
-botApp opts =
+botApp :: Options Unwrapped -> Wreq.Options -> BotApp Model Action
+botApp opts wreqOpts =
   BotApp
     { botInitialModel = (),
       botAction = action,
       botHandler = handler,
-      botJobs = [scrapeJob opts]
+      botJobs = [scrapeJob opts wreqOpts]
     }
 
 action :: Update -> Model -> Maybe Action
@@ -191,18 +196,18 @@ handler (StartChat chat) model =
   model <# do
     liftIO $ putStrLn $ "Chat started! " <> ppAsJSON chat
 
-scrapeJob :: Options Unwrapped -> BotJob Model Action
-scrapeJob opts =
+scrapeJob :: Options Unwrapped -> Wreq.Options -> BotJob Model Action
+scrapeJob opts wreqOpts =
   BotJob
     { botJobSchedule = "* * * * *",
-      botJobTask = scrapeJobTask opts
+      botJobTask = scrapeJobTask opts wreqOpts
     }
 
-scrapeJobTask :: Options Unwrapped -> Model -> Eff Action Model
-scrapeJobTask opts m =
+scrapeJobTask :: Options Unwrapped -> Wreq.Options -> Model -> Eff Action Model
+scrapeJobTask opts wreqOpts m =
   m <# do
     liftIO $ putStrLn "Starting scrape job"
-    res <- liftIO queryIBW
+    res <- liftIO $ queryIBW wreqOpts
     let offers = fromJust $ scrapeStringLike res.searchresults scrapeOffers
     liftIO $ putStrLn "Fetched offers"
     conn <- liftIO $ open (dbFile opts)

berlin-scraper/berlin-scraper.cabal

@@ -25,6 +25,8 @@ executable berlin-scraper
                     , optparse-generic
                     , telegram-bot-simple
                     , telegram-bot-api
+                    , servant-client
+                    , http-client-openssl
                     , bytestring
     hs-source-dirs:   app
     default-language: GHC2021

berlin-scraper/default.nix

@@ -1,6 +1,6 @@
-{ mkDerivation, aeson, base, bytestring, lens, lib
-, optparse-generic, scalpel, sqlite-simple, telegram-bot-api
-, telegram-bot-simple, text, wreq
+{ mkDerivation, aeson, base, bytestring, http-client-openssl, lens
+, lib, optparse-generic, scalpel, servant-client, sqlite-simple
+, telegram-bot-api, telegram-bot-simple, text, wreq
 }:
 mkDerivation {
   pname = "berlin-scraper";
@@ -9,8 +9,9 @@ mkDerivation {
   isLibrary = false;
   isExecutable = true;
   executableHaskellDepends = [
-    aeson base bytestring lens optparse-generic scalpel sqlite-simple
-    telegram-bot-api telegram-bot-simple text wreq
+    aeson base bytestring http-client-openssl lens optparse-generic
+    scalpel servant-client sqlite-simple telegram-bot-api
+    telegram-bot-simple text wreq
   ];
   license = lib.licenses.agpl3Plus;
   mainProgram = "berlin-scraper";