Better logs

berlin-scraper/app/Main.hs

@@ -11,10 +11,12 @@ import Control.Exception
 import Control.Lens (view, (&), (.~))
 import Control.Monad (when)
 import Control.Monad.IO.Class
+import Control.Monad.Trans.Class (lift)
 import Data.Aeson hiding (Options)
 import Data.Aeson qualified as Aeson
 import Data.ByteString.Lazy qualified as LBS
 import Data.Maybe (fromJust, fromMaybe, listToMaybe)
+import Data.String (IsString (..))
 import Data.Text qualified as Text
 import Data.Text.Encoding qualified as Text
 import Database.SQLite.Simple
@@ -31,7 +33,6 @@ import Telegram.Bot.API
 import Telegram.Bot.Simple
 import Telegram.Bot.Simple.Debug
 import Text.HTML.Scalpel
-import Data.String (IsString(..))
 
 main :: IO ()
 main = withOpenSSL $ do
@@ -50,7 +51,7 @@ main = withOpenSSL $ do
     liftIO $
       startBot (traceBotDefault $ botApp env) clientEnv >>= \case
         Left err -> do
-          info env "main" $ "Bot failed with: " <> fromString (show err)
+          runKatipContextT env.logEnv () "main" $ info $ "Bot failed with: " <> fromString (show err)
           exitFailure
         _ -> pure ()
 
@@ -92,8 +93,11 @@ data Env = Env
 
 -- * Parsing
 
-queryIBW :: Wreq.Options -> IO IBWResponse
-queryIBW wreqOpts = do
+queryIBW :: Wreq.Options -> WBS -> IO IBWResponse
+queryIBW wreqOpts wbs = do
+  let wbsParam = case wbs of
+        WBS -> "all"
+        NoWBS -> "0"
   let reqBody =
         [ partText "q" "wf-save-srch",
           partText "save" "false",
@@ -112,7 +116,7 @@ queryIBW wreqOpts = do
           partText "etagen_dg" "false",
           partText "balkon_loggia_terrasse" "false",
           partText "garten" "false",
-          partText "wbs" "all",
+          partText "wbs" wbsParam,
           partText "barrierefrei" "false",
           partText "gaeste_wc" "false",
           partText "aufzug" "false",
@@ -164,23 +168,30 @@ readEuropeanNumber x = do
 
 -- * SQLite
 
-insertOffer :: Connection -> Offer -> IO ()
-insertOffer conn =
-  execute conn "INSERT INTO offers (id, title, address, rooms, area, availableFrom, link) VALUES (?, ?, ?, ?, ?, ?, ?)"
+data WBS = WBS | NoWBS
 
-getOffer :: Connection -> Text -> IO (Maybe Offer)
-getOffer conn offerId =
-  listToMaybe <$> query conn "SELECT * from offers where id = ?" (Only offerId)
+tableName :: WBS -> Text
+tableName WBS = "offers"
+tableName NoWBS = "offers_without_wbs"
+
+insertOffer :: Connection -> WBS -> Offer -> IO ()
+insertOffer conn wbs =
+  execute conn $ "INSERT INTO " <> Query (tableName wbs) <> " (id, title, address, rooms, area, availableFrom, link) VALUES (?, ?, ?, ?, ?, ?, ?)"
+
+getOffer :: Connection -> WBS -> Text -> IO (Maybe Offer)
+getOffer conn wbs offerId =
+  listToMaybe <$> query conn ("SELECT * from " <> Query (tableName wbs) <> " where id = ?") (Only offerId)
 
 createTable :: Connection -> IO ()
-createTable conn =
+createTable conn = do
   execute_ conn "CREATE TABLE IF NOT EXISTS offers (id TEXT PRIMARY KEY, title TEXT, address TEXT, rooms REAL, area REAL, availableFrom TEXT, link TEXT)"
+  execute_ conn "CREATE TABLE IF NOT EXISTS offers_without_wbs (id TEXT PRIMARY KEY, title TEXT, address TEXT, rooms REAL, area REAL, availableFrom TEXT, link TEXT)"
 
-saveOffer :: Connection -> Offer -> IO Bool
-saveOffer conn offer = do
-  getOffer conn offer.id_ >>= \case
+saveOffer :: Connection -> WBS -> Offer -> IO Bool
+saveOffer conn wbs offer = do
+  getOffer conn wbs offer.id_ >>= \case
     Nothing -> do
-      insertOffer conn offer
+      insertOffer conn wbs offer
       pure True
     _ -> pure False
 
@@ -211,36 +222,48 @@ action update _ = do
 handler :: Env -> Action -> Model -> Eff Action Model
 handler env (StartChat chat) model =
   model <# do
-    info env "telegram.handler" $ "Chat started! " <> fromString (ppAsJSON chat)
+    runKatipContextT env.logEnv () "telegram.handler" $ info $ "Chat started! " <> fromString (ppAsJSON chat)
 
 scrapeJob :: Env -> BotJob Model Action
 scrapeJob env =
   BotJob
     { botJobSchedule = "* * * * *",
-      botJobTask = scrapeJobTask env
+      botJobTask = \m -> do
+        scrapeJobTask env m WBS
+        scrapeJobTask env m NoWBS
     }
 
-scrapeJobTask :: Env -> Model -> Eff Action Model
-scrapeJobTask env m =
-  m <# do
-    info env "scrapeJobTask" "Starting scrape job"
-    res <- liftIO $ queryIBW env.wreqOpts
-    let offers = fromJust $ scrapeStringLike res.searchresults scrapeOffers
-    info env "scrapeJobTask" "Fetched offers"
-    mapM_
-      ( \offer -> do
-          isNewOffer <- liftIO $ saveOffer env.dbConn offer
-          when isNewOffer $ do
-            info env "scrapeJobTask" "Found a new offer"
-            notify env offer
+wbsText :: WBS -> Text
+wbsText WBS = "WBS"
+wbsText NoWBS = "NoWBS"
+
+scrapeJobTask :: Env -> Model -> WBS -> Eff Action Model
+scrapeJobTask env m wbs =
+  m
+    <# runKatipContextT
+      env.logEnv
+      (sl "wbs" (wbsText wbs))
+      "scrapeJobTask"
+      ( do
+          info "Starting scrape job"
+          res <- liftIO $ queryIBW env.wreqOpts wbs
+          let offers = fromJust $ scrapeStringLike res.searchresults scrapeOffers
+          info "Fetched offers"
+          mapM_
+            ( \offer -> do
+                isNewOffer <- liftIO $ saveOffer env.dbConn wbs offer
+                when isNewOffer $ do
+                  info "Found a new offer"
+                  katipAddNamespace "notify" $ notify wbs offer
+            )
+            offers
       )
-      offers
 
-info :: MonadIO m => Env -> Namespace -> LogStr -> m ()
-info env ns l = liftIO $ runKatipT env.logEnv $ logMsg ns InfoS l
+info :: (Katip m) => LogStr -> m ()
+info = logMsg mempty InfoS
 
-notify :: Env -> Offer -> BotM ()
-notify env offer = do
+notify :: WBS -> Offer -> KatipContextT BotM ()
+notify wbs offer = do
   let offerTitle = "<b><u>" <> offer.title <> "</u></b>"
       offerAddress = "<b>Address:</b> " <> fromMaybe "N/A" offer.address
       offerRooms = "<b>Rooms:</b> " <> maybe "N/A" (Text.pack . show) offer.rooms
@@ -251,10 +274,17 @@ notify env offer = do
       -- sendMsgReq1 = (defSendMessage (SomeChatId $ ChatId 952512153) offerText) {sendMessageParseMode = Just HTML}
       -- sendMsgReq2 = (defSendMessage (SomeChatId $ ChatId 116981707) offerText) {sendMessageParseMode = Just HTML}
       sendMsgReq3 = (defSendMessage (SomeChatId $ ChatId 5781922807) offerText) {sendMessageParseMode = Just HTML}
-  res3 <- runTG sendMsgReq3
-  liftIO $
-    if res3.responseOk
-      then info env "notify" "Notified successfully"
-      else do
-        info env "notify" $ "Failed to notify the offer: " <> fromString (show offer)
-        info env "notify" $ "Response: " <> fromString (Text.unpack $ Text.decodeUtf8 (LBS.toStrict $ Aeson.encode res3))
+      sendMsgReq4 = (defSendMessage (SomeChatId $ ChatId 7008484163) offerText) {sendMessageParseMode = Just HTML}
+      msgReq = case wbs of
+        WBS -> sendMsgReq3
+        NoWBS -> sendMsgReq4
+  sendMessageWithLogs offer msgReq
+
+sendMessageWithLogs :: Offer -> SendMessageRequest -> KatipContextT BotM ()
+sendMessageWithLogs offer sendMsgReq = do
+  res <- lift $ runTG sendMsgReq
+  if res.responseOk
+    then info "Notified successfully"
+    else do
+      info $ "Failed to notify the offer: " <> fromString (show offer)
+      info $ "Response: " <> fromString (Text.unpack $ Text.decodeUtf8 (LBS.toStrict $ Aeson.encode res))

berlin-scraper/berlin-scraper.cabal

@@ -29,6 +29,7 @@ executable berlin-scraper
                     , http-client-openssl
                     , bytestring
                     , katip
+                    , transformers
     hs-source-dirs:   app
     default-language: GHC2021
     ghc-options:    -threaded -with-rtsopts=-N

berlin-scraper/default.nix

@@ -1,6 +1,7 @@
 { mkDerivation, aeson, base, bytestring, http-client-openssl, katip
 , lens, lib, optparse-generic, scalpel, servant-client
-, sqlite-simple, telegram-bot-api, telegram-bot-simple, text, wreq
+, sqlite-simple, telegram-bot-api, telegram-bot-simple, text
+, transformers, wreq
 }:
 mkDerivation {
   pname = "berlin-scraper";
@@ -11,7 +12,7 @@ mkDerivation {
   executableHaskellDepends = [
     aeson base bytestring http-client-openssl katip lens
     optparse-generic scalpel servant-client sqlite-simple
-    telegram-bot-api telegram-bot-simple text wreq
+    telegram-bot-api telegram-bot-simple text transformers wreq
   ];
   license = lib.licenses.agpl3Plus;
   mainProgram = "berlin-scraper";