From 96546a8f7eb71e5e259fb659442f2e46a0052d4a Mon Sep 17 00:00:00 2001 From: towards-a-new-leftypol Date: Thu, 25 Jan 2024 23:28:59 -0500 Subject: [PATCH] Fix thumbnail paths --- sql/initialize.sql | 40 ++++++++++++++++++++++++++-------------- src/Backfill.hs | 30 ++++++++++++++++++------------ src/Common | 2 +- src/JSONCommonTypes.hs | 2 +- 4 files changed, 46 insertions(+), 28 deletions(-) diff --git a/sql/initialize.sql b/sql/initialize.sql index 1393ac1..c0def53 100644 --- a/sql/initialize.sql +++ b/sql/initialize.sql @@ -116,6 +116,7 @@ CREATE TABLE IF NOT EXISTS attachments , post_id bigint NOT NULL , resolution dimension , file_extension text + , thumb_extension text , original_filename text , board_filename text NOT NULL , spoiler boolean NOT NULL DEFAULT true @@ -215,7 +216,7 @@ $$; CREATE OR REPLACE FUNCTION fetch_catalog(max_time timestamptz, max_row_read int DEFAULT 10000) RETURNS TABLE ( - post_count bigint, + -- post_count bigint, estimated_post_count bigint, post_id bigint, board_post_id bigint, @@ -227,7 +228,12 @@ RETURNS TABLE ( board_thread_id bigint, pathpart text, site_name text, - site_id int + file_mimetype text, + file_illegal boolean, + -- file_resolution dimension, + file_name text, + file_extension text, + file_thumb_extension text ) AS $$ WITH top AS @@ -255,30 +261,36 @@ RETURNS TABLE ( * FROM tall_posts t ORDER BY t.thread_id, t.board_post_id - ), - post_counts AS - ( - SELECT thread_id, count(*) as post_count FROM - tall_posts - GROUP BY thread_id - ) + )-- , + -- post_counts AS + -- ( + -- SELECT thread_id, count(*) as post_count FROM + -- tall_posts + -- GROUP BY thread_id + -- ) SELECT - post_counts.post_count, + -- post_counts.post_count, op_posts.*, - threads.board_thread_id, + threads.board_thread_id, -- this should be part of the url path when creating links, not thread_id (that's internal) boards.pathpart, sites."name", - sites.site_id + -- sites.site_id, + attachments.mimetype as file_mimetype, + attachments.illegal as file_illegal, + -- attachments.resolution as file_resolution, + attachments.board_filename as file_name, + attachments.file_extension, + attachments.thumb_extension as file_thumb_extension FROM op_posts - JOIN post_counts ON op_posts.thread_id = post_counts.thread_id + -- JOIN post_counts ON op_posts.thread_id = post_counts.thread_id JOIN threads ON op_posts.thread_id = threads.thread_id JOIN boards ON threads.board_id = boards.board_id JOIN sites ON sites.site_id = boards.site_id + LEFT OUTER JOIN attachments ON attachments.post_id = op_posts.post_id ORDER BY bump_time DESC; $$ LANGUAGE sql; - -- Function: search_posts -- -- This function performs a full-text search on the `posts` table using PostgreSQL's text search features. diff --git a/src/Backfill.hs b/src/Backfill.hs index 06f56e9..866cc2e 100644 --- a/src/Backfill.hs +++ b/src/Backfill.hs @@ -17,7 +17,7 @@ import System.Directory , copyFile , createDirectoryIfMissing ) -import System.FilePath (()) +import System.FilePath ((), (<.>), takeExtension) import Data.List (find, isSuffixOf, foldl') import qualified Data.Set as Set import Data.Set (Set) @@ -254,15 +254,18 @@ fileToAttachment post file = , At.post_id = fromJust $ Posts.post_id post , At.resolution = dim , At.file_extension = Just extension + , At.thumb_extension = Just thumb_extension , At.original_filename = Just $ JS.filename file <> "." <> extension , At.file_size_bytes = JS.fsize file - , At.board_filename = JS.id file <> "." <> extension - , At.spoiler = JS.spoiler file + , At.board_filename = JS.id file + , At.spoiler = maybe False id $ JS.spoiler file } where extension = JS.ext file + thumb_extension = T.pack $ drop 1 $ takeExtension $ unpack $ JS.thumb_path file + guessed_mime = getMimeType extension dim = (JS.w file) >>= \w -> @@ -293,13 +296,13 @@ copyFiles settings (site, board, thread, _, path, attachment) = do createDirectoryIfMissing True common_dest if src_exists - then copyFile src dest + then putStrLn ("Copying " ++ src) >> copyFile src dest else return () thumb_exists <- doesFileExist thumb_src if thumb_exists - then copyFile thumb_src thumb_dest + then putStrLn ("Copying " ++ thumb_src) >> copyFile thumb_src thumb_dest else return () else return () @@ -312,22 +315,24 @@ copyFiles settings (site, board, thread, _, path, attachment) = do src = At.file_path path thumb_src :: FilePath - thumb_src = At.file_path path + thumb_src = At.thumbnail_path path dest :: FilePath dest = common_dest - <> "/" <> (unpack $ At.board_filename attachment) + (unpack $ At.board_filename attachment) + <.> (unpack $ fromJust $ At.file_extension attachment) thumb_dest :: FilePath thumb_dest = common_dest - <> "/thumbnail_" <> (unpack $ At.board_filename attachment) + "thumbnail_" <> (unpack $ At.board_filename attachment) + <.> (unpack $ fromJust $ At.thumb_extension attachment) common_dest :: FilePath common_dest = (JSONSettings.media_root_path settings) - <> "/" <> Sites.name site - <> "/" <> Boards.pathpart board - <> "/" <> (show $ Threads.board_thread_id thread) + Sites.name site + Boards.pathpart board + (show $ Threads.board_thread_id thread) processFiles :: JSONSettings -> [(Sites.Site, Boards.Board, Threads.Thread, JSONPosts.Post, Posts.Post)] -> IO () @@ -449,9 +454,10 @@ processFiles settings tuples = do -- perfect just means that our posts have ids, , At.post_id = undefined , At.resolution = undefined , At.file_extension = Just $ T.drop 1 ext + , At.thumb_extension = Just $ "png" , At.original_filename = Just $ filename <> ext , At.file_size_bytes = size - , At.board_filename = tim <> ext + , At.board_filename = tim , At.spoiler = spoiler > 0 } diff --git a/src/Common b/src/Common index 7898fb7..3f420d3 160000 --- a/src/Common +++ b/src/Common @@ -1 +1 @@ -Subproject commit 7898fb7a15e57c093ba32e8385cb33683e1a0a30 +Subproject commit 3f420d3131eb0477b522103a582732a5ba365f19 diff --git a/src/JSONCommonTypes.hs b/src/JSONCommonTypes.hs index 07a904d..832d4b3 100644 --- a/src/JSONCommonTypes.hs +++ b/src/JSONCommonTypes.hs @@ -29,7 +29,7 @@ data File = File , w :: Maybe Int , fsize :: Int , filename :: Text - , spoiler :: Bool + , spoiler :: Maybe Bool , md5 :: Text , file_path :: Text , thumb_path :: Text