{-# LANGUAGE BlockArguments #-} {-# LANGUAGE DataKinds #-} {-# LANGUAGE LambdaCase #-} {-# LANGUAGE OverloadedRecordDot #-} {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE RecordWildCards #-} module Main where import Control.Lens (view, (&), (.~)) import Control.Monad (when) import Control.Monad.IO.Class import Data.Aeson hiding (Options) import Data.Aeson qualified as Aeson import Data.ByteString.Lazy qualified as LBS import Data.Maybe (fromJust, fromMaybe, listToMaybe) import Data.Text qualified as Text import Data.Text.Encoding qualified as Text import Data.Text.IO qualified as Text import Database.SQLite.Simple import GHC.Generics import Network.HTTP.Client.OpenSSL import Network.Wreq hiding (Options) import Network.Wreq qualified as Wreq import Options.Generic import Servant.Client (mkClientEnv) import System.Exit (exitFailure) import Telegram.Bot.API import Telegram.Bot.Simple import Telegram.Bot.Simple.Debug import Text.HTML.Scalpel main :: IO () main = withOpenSSL $ do opts :: Options Unwrapped <- unwrapRecord "Berlin Scrapper" token <- getEnvToken "TELEGRAM_BOT_TOKEN" httpMgr <- newOpenSSLManager dbConn <- open opts.dbFile createTable dbConn let clientEnv = mkClientEnv httpMgr (botBaseUrl token) wreqOpts = Wreq.defaults & Wreq.manager .~ Right httpMgr env = Env {..} startBot (traceBotDefault $ botApp env) clientEnv >>= \case Left err -> do putStrLn $ "Bot failed with: " <> show err exitFailure _ -> pure () -- * Types newtype Options w = Options {dbFile :: w ::: FilePath "Path to the sqlite database"} deriving (Generic) instance ParseRecord (Options Wrapped) data Offer = Offer { id_ :: Text, title :: Text, address :: Maybe Text, rooms :: Maybe Double, area :: Maybe Double, availableFrom :: Maybe Text, link :: Text } deriving (Show, Generic) instance FromRow Offer instance ToRow Offer data IBWResponse = IBWResponse { headline :: Text, searchresults :: Text } deriving (Generic) instance FromJSON IBWResponse data Env = Env { dbConn :: Connection, wreqOpts :: Wreq.Options } -- * Parsing queryIBW :: Wreq.Options -> IO IBWResponse queryIBW wreqOpts = do let reqBody = [ partText "q" "wf-save-srch", partText "save" "false", partText "heizung_zentral" "false", partText "heizung_etage" "false", partText "energy_fernwaerme" "false", partText "heizung_nachtstrom" "false", partText "heizung_ofen" "false", partText "heizung_gas" "false", partText "heizung_oel" "false", partText "heizung_solar" "false", partText "heizung_erdwaerme" "false", partText "heizung_fussboden" "false", partText "seniorenwohnung" "false", partText "maisonette" "false", partText "etagen_dg" "false", partText "balkon_loggia_terrasse" "false", partText "garten" "false", partText "wbs" "0", partText "barrierefrei" "false", partText "gaeste_wc" "false", partText "aufzug" "false", partText "stellplatz" "false", partText "keller" "false", partText "badewanne" "false", partText "dusche" "false", partText "bez[]" "01_00", partText "bez[]" "02_00", partText "bez[]" "04_00", partText "bez[]" "07_00", partText "bez[]" "08_00", partText "bez[]" "09_00", partText "bez[]" "11_00" ] let link = "https://inberlinwohnen.de/wp-content/themes/ibw/skript/wohnungsfinder.php" resBS <- view responseBody <$> postWith wreqOpts link reqBody pure $ fromJust $ decode @IBWResponse resBS scrapeOffers :: Scraper Text [Offer] scrapeOffers = do let listItemsSelector = "div" @: [hasClass "result-list"] // "ul" // "li" chroots listItemsSelector scrapeOffer scrapeOffer :: Scraper Text Offer scrapeOffer = do title <- Text.strip <$> text "h3" let tableSelector = "div" @: [hasClass "tb-merkdetails"] // "div" @: [hasClass "span_wflist_data"] // "table" @: [hasClass "fullw"] // "tbody" rowsSelector = tableSelector // "tr" tableDataParser = do thVal <- text "th" (Text.strip thVal,) <$> text "td" allTableData <- chroots rowsSelector tableDataParser let address = lookup "Adresse:" allTableData roomsStr = lookup "Zimmeranzahl:" allTableData rooms = readEuropeanNumber <$> roomsStr areaStr = lookup "Wohnfläche:" allTableData area = readEuropeanNumber <$> (Text.stripSuffix " m²" =<< areaStr) availableFrom = lookup "Bezugsfertig ab:" allTableData link <- attr "href" $ "a" @: [hasClass "org-but"] id_ <- attr "id" "li" pure Offer {..} readEuropeanNumber :: Text -> Double readEuropeanNumber x = do read $ Text.unpack $ Text.replace "," "." x -- * SQLite insertOffer :: Connection -> Offer -> IO () insertOffer conn = execute conn "INSERT INTO offers (id, title, address, rooms, area, availableFrom, link) VALUES (?, ?, ?, ?, ?, ?, ?)" getOffer :: Connection -> Text -> IO (Maybe Offer) getOffer conn offerId = listToMaybe <$> query conn "SELECT * from offers where id = ?" (Only offerId) createTable :: Connection -> IO () createTable conn = execute_ conn "CREATE TABLE IF NOT EXISTS offers (id TEXT PRIMARY KEY, title TEXT, address TEXT, rooms REAL, area REAL, availableFrom TEXT, link TEXT)" saveOffer :: Connection -> Offer -> IO Bool saveOffer conn offer = do getOffer conn offer.id_ >>= \case Nothing -> do insertOffer conn offer pure True _ -> pure False -- * Telegram type Model = () newtype Action = StartChat Chat deriving (Show) botApp :: Env -> BotApp Model Action botApp env = BotApp { botInitialModel = (), botAction = action, botHandler = handler, botJobs = [scrapeJob env] } action :: Update -> Model -> Maybe Action action update _ = do msg <- update.updateMessage txt <- msg.messageText if txt == "/start" then pure $ StartChat msg.messageChat else Nothing handler :: Action -> Model -> Eff Action Model handler (StartChat chat) model = model <# do liftIO $ putStrLn $ "Chat started! " <> ppAsJSON chat scrapeJob :: Env -> BotJob Model Action scrapeJob env = BotJob { botJobSchedule = "* * * * *", botJobTask = scrapeJobTask env } scrapeJobTask :: Env -> Model -> Eff Action Model scrapeJobTask env m = m <# do liftIO $ putStrLn "Starting scrape job" res <- liftIO $ queryIBW env.wreqOpts let offers = fromJust $ scrapeStringLike res.searchresults scrapeOffers liftIO $ putStrLn "Fetched offers" mapM_ ( \offer -> do isNewOffer <- liftIO $ saveOffer env.dbConn offer when isNewOffer $ do liftIO $ putStrLn "Found a new offer" notify offer ) offers notify :: Offer -> BotM () notify offer = do let offerTitle = "" <> offer.title <> "" offerAddress = "Address: " <> fromMaybe "N/A" offer.address offerRooms = "Rooms: " <> maybe "N/A" (Text.pack . show) offer.rooms offerArea = "Area: " <> maybe "N/A" ((<> " m²") . Text.pack . show) offer.area offerLink = " offer.link <> "\" >Apply Here" offerBody = offerAddress <> "\n" <> offerRooms <> "\n" <> offerArea <> "\n" <> offerLink offerText = offerTitle <> "\n\n" <> offerBody sendMsgReq1 = (defSendMessage (SomeChatId $ ChatId 952512153) offerText) {sendMessageParseMode = Just HTML} sendMsgReq2 = (defSendMessage (SomeChatId $ ChatId 116981707) offerText) {sendMessageParseMode = Just HTML} res1 <- runTG sendMsgReq1 liftIO $ if res1.responseOk then putStrLn "Notified successfully" else do putStrLn $ "Failed to notify the offer: " <> show offer Text.putStrLn $ "Response: " <> Text.decodeUtf8 (LBS.toStrict $ Aeson.encode res1) res2 <- runTG sendMsgReq2 liftIO $ if res2.responseOk then putStrLn "Notified successfully" else do putStrLn $ "Failed to notify the offer: " <> show offer Text.putStrLn $ "Response: " <> Text.decodeUtf8 (LBS.toStrict $ Aeson.encode res2)