diff --git a/berlin-scraper/app/Main.hs b/berlin-scraper/app/Main.hs index 5ac6eaa..3df80f3 100644 --- a/berlin-scraper/app/Main.hs +++ b/berlin-scraper/app/Main.hs @@ -7,7 +7,7 @@ module Main where -import Control.Lens (view) +import Control.Lens (view, (&), (.~)) import Control.Monad (when) import Control.Monad.IO.Class import Data.Aeson hiding (Options) @@ -19,20 +19,25 @@ import Data.Text.Encoding qualified as Text import Data.Text.IO qualified as Text import Database.SQLite.Simple import GHC.Generics +import Network.HTTP.Client.OpenSSL import Network.Wreq hiding (Options) import Options.Generic +import Servant.Client (mkClientEnv) import System.Exit (exitFailure) import Telegram.Bot.API import Telegram.Bot.Simple import Telegram.Bot.Simple.Debug import Text.HTML.Scalpel +import qualified Network.Wreq as Wreq main :: IO () -main = do +main = withOpenSSL $ do opts <- unwrapRecord "Berlin Scrapper" token <- getEnvToken "TELEGRAM_BOT_TOKEN" - clientEnv <- defaultTelegramClientEnv token - startBot (traceBotDefault $ botApp opts) clientEnv >>= \case + httpMgr <- newOpenSSLManager + let clientEnv = mkClientEnv httpMgr (botBaseUrl token) + wreqOpts = Wreq.defaults & Wreq.manager .~ Right httpMgr + startBot (traceBotDefault $ botApp opts wreqOpts) clientEnv >>= \case Left err -> do putStrLn $ "Bot failed with: " <> show err exitFailure @@ -70,8 +75,8 @@ instance FromJSON IBWResponse -- * Parsing -queryIBW :: IO IBWResponse -queryIBW = do +queryIBW :: Wreq.Options -> IO IBWResponse +queryIBW wreqOpts = do let reqBody = [ partText "q" "wf-save-srch", partText "save" "false", @@ -107,7 +112,7 @@ queryIBW = do partText "bez[]" "11_00" ] let link = "https://inberlinwohnen.de/wp-content/themes/ibw/skript/wohnungsfinder.php" - resBS <- view responseBody <$> post link reqBody + resBS <- view responseBody <$> postWith wreqOpts link reqBody pure $ fromJust $ decode @IBWResponse resBS scrapeOffers :: Scraper Text [Offer] @@ -169,13 +174,13 @@ type Model = () newtype Action = StartChat Chat deriving (Show) -botApp :: Options Unwrapped -> BotApp Model Action -botApp opts = +botApp :: Options Unwrapped -> Wreq.Options -> BotApp Model Action +botApp opts wreqOpts = BotApp { botInitialModel = (), botAction = action, botHandler = handler, - botJobs = [scrapeJob opts] + botJobs = [scrapeJob opts wreqOpts] } action :: Update -> Model -> Maybe Action @@ -191,18 +196,18 @@ handler (StartChat chat) model = model <# do liftIO $ putStrLn $ "Chat started! " <> ppAsJSON chat -scrapeJob :: Options Unwrapped -> BotJob Model Action -scrapeJob opts = +scrapeJob :: Options Unwrapped -> Wreq.Options -> BotJob Model Action +scrapeJob opts wreqOpts = BotJob { botJobSchedule = "* * * * *", - botJobTask = scrapeJobTask opts + botJobTask = scrapeJobTask opts wreqOpts } -scrapeJobTask :: Options Unwrapped -> Model -> Eff Action Model -scrapeJobTask opts m = +scrapeJobTask :: Options Unwrapped -> Wreq.Options -> Model -> Eff Action Model +scrapeJobTask opts wreqOpts m = m <# do liftIO $ putStrLn "Starting scrape job" - res <- liftIO queryIBW + res <- liftIO $ queryIBW wreqOpts let offers = fromJust $ scrapeStringLike res.searchresults scrapeOffers liftIO $ putStrLn "Fetched offers" conn <- liftIO $ open (dbFile opts) diff --git a/berlin-scraper/berlin-scraper.cabal b/berlin-scraper/berlin-scraper.cabal index 2f51f60..59c1b35 100644 --- a/berlin-scraper/berlin-scraper.cabal +++ b/berlin-scraper/berlin-scraper.cabal @@ -25,6 +25,8 @@ executable berlin-scraper , optparse-generic , telegram-bot-simple , telegram-bot-api + , servant-client + , http-client-openssl , bytestring hs-source-dirs: app default-language: GHC2021 diff --git a/berlin-scraper/default.nix b/berlin-scraper/default.nix index e85f4e5..1d2a1e4 100644 --- a/berlin-scraper/default.nix +++ b/berlin-scraper/default.nix @@ -1,6 +1,6 @@ -{ mkDerivation, aeson, base, bytestring, lens, lib -, optparse-generic, scalpel, sqlite-simple, telegram-bot-api -, telegram-bot-simple, text, wreq +{ mkDerivation, aeson, base, bytestring, http-client-openssl, lens +, lib, optparse-generic, scalpel, servant-client, sqlite-simple +, telegram-bot-api, telegram-bot-simple, text, wreq }: mkDerivation { pname = "berlin-scraper"; @@ -9,8 +9,9 @@ mkDerivation { isLibrary = false; isExecutable = true; executableHaskellDepends = [ - aeson base bytestring lens optparse-generic scalpel sqlite-simple - telegram-bot-api telegram-bot-simple text wreq + aeson base bytestring http-client-openssl lens optparse-generic + scalpel servant-client sqlite-simple telegram-bot-api + telegram-bot-simple text wreq ]; license = lib.licenses.agpl3Plus; mainProgram = "berlin-scraper";