diff --git a/chan-delorean.cabal b/chan-delorean.cabal index 9cf2f4f..0f7b264 100644 --- a/chan-delorean.cabal +++ b/chan-delorean.cabal @@ -80,6 +80,7 @@ executable chan-delorean Data.WordUtil Network.DataClient Network.DataClientTypes + Network.GetLatestPostsPerBoardResponse Common.Server.JSONSettings Common.Server.ConsumerSettings diff --git a/consumer_settings.json b/consumer_settings.json index 352b0ff..0a223af 100644 --- a/consumer_settings.json +++ b/consumer_settings.json @@ -13,5 +13,5 @@ "jwt": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyb2xlIjoiY2hhbl9hcmNoaXZlciJ9.rGIKZokTDKTuQLIv8138bUby5PELfDipYYIDpJzH02c", "media_root_path": "/home/phil/linixy/tmp/chan_archive_media_repaired/archive", "http_fill_all": false, - "http_sync_continously": false + "http_sync_continously": true } diff --git a/sql/archive_tests2.sql b/sql/archive_tests2.sql index cf43e43..fb178d5 100644 --- a/sql/archive_tests2.sql +++ b/sql/archive_tests2.sql @@ -290,6 +290,7 @@ SELECT DISTINCT ON (b.board_id) JOIN posts p ON p.thread_id = t.thread_id ORDER BY b.board_id, p.creation_time DESC; + CREATE OR REPLACE FUNCTION get_latest_posts_per_board() RETURNS TABLE ( board_id int, @@ -298,7 +299,6 @@ RETURNS TABLE ( post_id bigint, board_post_id bigint, creation_time timestamp with time zone, - body text, thread_id bigint, board_thread_id bigint ) AS $$ @@ -309,7 +309,6 @@ RETURNS TABLE ( p.post_id, p.board_post_id, p.creation_time, - p.body, t.thread_id, t.board_thread_id FROM boards b @@ -318,4 +317,4 @@ RETURNS TABLE ( ORDER BY b.board_id, p.creation_time DESC; $$ LANGUAGE sql STABLE; -SELECT * FROM get_latest_posts_per_board(); \ No newline at end of file +SELECT * FROM get_latest_posts_per_board(); diff --git a/sql/initialize.sql b/sql/initialize.sql index c3024a9..370b5c0 100644 --- a/sql/initialize.sql +++ b/sql/initialize.sql @@ -409,7 +409,6 @@ RETURNS TABLE ( post_id bigint, board_post_id bigint, creation_time timestamp with time zone, - body text, thread_id bigint, board_thread_id bigint ) AS $$ @@ -420,7 +419,6 @@ RETURNS TABLE ( p.post_id, p.board_post_id, p.creation_time, - p.body, t.thread_id, t.board_thread_id FROM boards b diff --git a/src/Network/DataClient.hs b/src/Network/DataClient.hs index c7b3879..163fbfe 100644 --- a/src/Network/DataClient.hs +++ b/src/Network/DataClient.hs @@ -1,5 +1,7 @@ {-# LANGUAGE OverloadedStrings #-} {-# LANGUAGE DeriveAnyClass #-} +{-# OPTIONS_GHC -Wno-unrecognised-pragmas #-} +{-# HLINT ignore "Use <&>" #-} module Network.DataClient ( HttpError(..) @@ -19,6 +21,7 @@ module Network.DataClient , postAttachments , getJSON , getFile + , getLatestPostsPerBoard ) where import Control.Monad (forM) @@ -49,6 +52,7 @@ import qualified Common.AttachmentType as Attachments import qualified Common.PostsType as Posts import Common.Network.HttpClient import qualified Network.DataClientTypes as T +import qualified Network.GetLatestPostsPerBoardResponse as GLPPBR data PostId = PostId @@ -233,14 +237,20 @@ getFile url = do case result of Left (err :: HttpError) -> do putStrLn $ "getFile " ++ url ++ " Error!" - putStrLn $ show err + print err return Nothing Right lbs -> do putStrLn $ "getFile " ++ url ++ " SUCCESS!" tmp_root <- getCanonicalTemporaryDirectory (tmp_filepath, tmp_filehandle) <- openBinaryTempFile tmp_root "chan.attachment" putStrLn $ "Created " ++ tmp_filepath - putStrLn $ "Writing attachment..." + putStrLn "Writing attachment..." LBS.hPut tmp_filehandle lbs hClose tmp_filehandle return $ Just tmp_filepath + + +-- | Function to handle each chunk. +getLatestPostsPerBoard :: T.JSONSettings -> IO (Either HttpError [ GLPPBR.GetLatestPostsPerBoardResponse ]) +getLatestPostsPerBoard settings = + post settings "/rpc/get_latest_posts_per_board" mempty False >>= return . eitherDecodeResponse diff --git a/src/Network/DataClientTypes.hs b/src/Network/DataClientTypes.hs index c74d0e2..bde9b51 100644 --- a/src/Network/DataClientTypes.hs +++ b/src/Network/DataClientTypes.hs @@ -10,4 +10,3 @@ data ThreadMaxIdx = ThreadMaxIdx { thread_id :: Int64 , max_idx :: Int } deriving (Show, Generic, FromJSON) - diff --git a/src/Network/GetLatestPostsPerBoardResponse.hs b/src/Network/GetLatestPostsPerBoardResponse.hs new file mode 100644 index 0000000..aecbfd9 --- /dev/null +++ b/src/Network/GetLatestPostsPerBoardResponse.hs @@ -0,0 +1,20 @@ +{-# LANGUAGE DeriveAnyClass #-} + +module Network.GetLatestPostsPerBoardResponse +where + +import Data.Int (Int64) +import Data.Time.Clock (UTCTime) +import Data.Aeson (FromJSON) +import GHC.Generics + +data GetLatestPostsPerBoardResponse = GetLatestPostsPerBoardResponse + { board_id :: Int + , site_id :: Int + , pathpart :: String + , post_id :: Maybe Int64 + , board_post_id :: Int64 + , creation_time :: UTCTime + , thread_id :: Int64 + , board_thread_id :: Integer + } deriving (Show, Generic, FromJSON) diff --git a/src/Sync.hs b/src/Sync.hs index 34f02e6..a4528c8 100644 --- a/src/Sync.hs +++ b/src/Sync.hs @@ -1,21 +1,31 @@ +{-# LANGUAGE RecordWildCards #-} + module Sync where -import Common.Server.ConsumerSettings -import Lib (getBoards, toClientSettings) -import SitesType (Site) -import BoardsType (Board) +import Common.Server.ConsumerSettings as Settings +import Common.Server.JSONSettings as JSONSettings +import Network.DataClient (getLatestPostsPerBoard) -getSiteBoards :: ConsumerJSONSettings -> JSONSiteSettings -> IO (Site, [ Board ]) -getSiteBoards settings site_settings = - let client_settings = toClientSettings settings site_settings - in getBoards - client_settings - (boards site_settings) +consumerSettingsToPartialJSONSettings :: Settings.ConsumerJSONSettings -> JSONSettings.JSONSettings +consumerSettingsToPartialJSONSettings ConsumerJSONSettings {..} = + JSONSettings + { JSONSettings.postgrest_url = postgrest_url + , JSONSettings.jwt = jwt + , backup_read_root = undefined + , JSONSettings.media_root_path + , site_name = undefined + , site_url = undefined + } syncWebsites :: ConsumerJSONSettings -> IO () -syncWebsites _ = do +syncWebsites consumer_settings = do putStrLn "Starting channel web synchronization." + let json_settings = consumerSettingsToPartialJSONSettings consumer_settings + + asdf <- getLatestPostsPerBoard json_settings + + print asdf -- first we need all the (Site, Board) tuples -- perhaps we even want all (Site, Board, Thread) pairs -- But then we don't load the posts of each thread, instead only do