Sync starts by getting latest post for each board

chan-delorean.cabal

@@ -80,6 +80,7 @@ executable chan-delorean
                       Data.WordUtil
                       Network.DataClient
                       Network.DataClientTypes
+                      Network.GetLatestPostsPerBoardResponse
                       Common.Server.JSONSettings
                       Common.Server.ConsumerSettings
 

consumer_settings.json

@@ -13,5 +13,5 @@
   "jwt": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiY2hhbl9hcmNoaXZlciJ9.rGIKZokTDKTuQLIv8138bUby5PELfDipYYIDpJzH02c",
   "media_root_path": "/home/phil/linixy/tmp/chan_archive_media_repaired/archive",
   "http_fill_all": false,
-  "http_sync_continously": false
+  "http_sync_continously": true
 }

sql/archive_tests2.sql

@@ -290,6 +290,7 @@ SELECT DISTINCT ON (b.board_id)
   JOIN posts   p ON p.thread_id = t.thread_id
  ORDER BY b.board_id, p.creation_time DESC;
 
+
 CREATE OR REPLACE FUNCTION get_latest_posts_per_board()
 RETURNS TABLE (
     board_id int,
@@ -298,7 +299,6 @@ RETURNS TABLE (
     post_id bigint,
     board_post_id bigint,
     creation_time timestamp with time zone,
-    body text,
     thread_id bigint,
     board_thread_id bigint
 ) AS $$
@@ -309,7 +309,6 @@ RETURNS TABLE (
            p.post_id,
            p.board_post_id,
            p.creation_time,
-           p.body,
            t.thread_id,
            t.board_thread_id
       FROM boards b
@@ -318,4 +317,4 @@ RETURNS TABLE (
      ORDER BY b.board_id, p.creation_time DESC;
 $$ LANGUAGE sql STABLE;
 
-SELECT * FROM get_latest_posts_per_board();
+SELECT * FROM get_latest_posts_per_board();

sql/initialize.sql

@@ -409,7 +409,6 @@ RETURNS TABLE (
     post_id bigint,
     board_post_id bigint,
     creation_time timestamp with time zone,
-    body text,
     thread_id bigint,
     board_thread_id bigint
 ) AS $$
@@ -420,7 +419,6 @@ RETURNS TABLE (
            p.post_id,
            p.board_post_id,
            p.creation_time,
-           p.body,
            t.thread_id,
            t.board_thread_id
       FROM boards b

src/Network/DataClient.hs

@@ -1,5 +1,7 @@
 {-# LANGUAGE OverloadedStrings #-}
 {-# LANGUAGE DeriveAnyClass #-}
+{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-}
+{-# HLINT ignore "Use <&>" #-}
 
 module Network.DataClient
   ( HttpError(..)
@@ -19,6 +21,7 @@ module Network.DataClient
   , postAttachments
   , getJSON
   , getFile
+  , getLatestPostsPerBoard
   ) where
 
 import Control.Monad (forM)
@@ -49,6 +52,7 @@ import qualified Common.AttachmentType as Attachments
 import qualified Common.PostsType  as Posts
 import Common.Network.HttpClient
 import qualified Network.DataClientTypes as T
+import qualified Network.GetLatestPostsPerBoardResponse as GLPPBR
 
 
 data PostId = PostId
@@ -233,14 +237,20 @@ getFile url = do
     case result of
         Left (err :: HttpError) -> do
             putStrLn $ "getFile " ++ url ++ " Error!"
-            putStrLn $ show err
+            print err
             return Nothing
         Right lbs -> do
             putStrLn $ "getFile " ++ url ++ " SUCCESS!"
             tmp_root <- getCanonicalTemporaryDirectory
             (tmp_filepath, tmp_filehandle) <- openBinaryTempFile tmp_root "chan.attachment"
             putStrLn $ "Created " ++ tmp_filepath
-            putStrLn $ "Writing attachment..."
+            putStrLn "Writing attachment..."
             LBS.hPut tmp_filehandle lbs
             hClose tmp_filehandle
             return $ Just tmp_filepath
+
+
+-- | Function to handle each chunk.
+getLatestPostsPerBoard :: T.JSONSettings -> IO (Either HttpError [ GLPPBR.GetLatestPostsPerBoardResponse ])
+getLatestPostsPerBoard settings =
+    post settings "/rpc/get_latest_posts_per_board" mempty False >>= return . eitherDecodeResponse

src/Network/DataClientTypes.hs

@@ -10,4 +10,3 @@ data ThreadMaxIdx = ThreadMaxIdx
     { thread_id :: Int64
     , max_idx   :: Int
     } deriving (Show, Generic, FromJSON)
-

src/Network/GetLatestPostsPerBoardResponse.hs

@@ -0,0 +1,20 @@
+{-# LANGUAGE DeriveAnyClass #-}
+
+module Network.GetLatestPostsPerBoardResponse
+where
+
+import Data.Int (Int64)
+import Data.Time.Clock (UTCTime)
+import Data.Aeson (FromJSON)
+import GHC.Generics
+
+data GetLatestPostsPerBoardResponse = GetLatestPostsPerBoardResponse
+    { board_id        :: Int
+    , site_id         :: Int
+    , pathpart        :: String
+    , post_id         :: Maybe Int64
+    , board_post_id   :: Int64
+    , creation_time   :: UTCTime
+    , thread_id       :: Int64
+    , board_thread_id :: Integer
+    } deriving (Show, Generic, FromJSON)

src/Sync.hs

@@ -1,21 +1,31 @@
+{-# LANGUAGE RecordWildCards #-}
+
 module Sync where
 
-import Common.Server.ConsumerSettings
-import Lib (getBoards, toClientSettings)
-import SitesType (Site)
-import BoardsType (Board)
+import Common.Server.ConsumerSettings as Settings
+import Common.Server.JSONSettings as JSONSettings
+import Network.DataClient (getLatestPostsPerBoard)
 
-getSiteBoards :: ConsumerJSONSettings -> JSONSiteSettings -> IO (Site, [ Board ])
-getSiteBoards settings site_settings =
-    let client_settings = toClientSettings settings site_settings
-    in getBoards
-        client_settings
-        (boards site_settings)
+consumerSettingsToPartialJSONSettings :: Settings.ConsumerJSONSettings -> JSONSettings.JSONSettings
+consumerSettingsToPartialJSONSettings ConsumerJSONSettings {..} =
+    JSONSettings
+        { JSONSettings.postgrest_url = postgrest_url
+        , JSONSettings.jwt = jwt
+        , backup_read_root = undefined
+        , JSONSettings.media_root_path
+        , site_name = undefined
+        , site_url = undefined
+        }
 
 syncWebsites :: ConsumerJSONSettings -> IO ()
-syncWebsites _ = do
+syncWebsites consumer_settings = do
     putStrLn "Starting channel web synchronization."
 
+    let json_settings = consumerSettingsToPartialJSONSettings consumer_settings
+
+    asdf <- getLatestPostsPerBoard json_settings
+
+    print asdf
     -- first we need all the (Site, Board) tuples
     -- perhaps we even want all (Site, Board, Thread) pairs
     -- But then we don't load the posts of each thread, instead only do