From 8bb7f413bf9e7a986d12d3f54d7aa45ba2753530 Mon Sep 17 00:00:00 2001 From: towards-a-new-leftypol Date: Mon, 26 Feb 2024 18:20:30 -0500 Subject: [PATCH] Add embed field - alternative post insert function to set the embed field on existing posts that don't have one --- sql/initialize.sql | 6 ++- sql/remake_fetch_catalog.sql | 4 +- sql/remake_insert_posts_and_insert_ids.sql | 44 ++++++++++++++++++++++ src/Backfill.hs | 6 ++- src/Common | 2 +- src/JSONPost.hs | 1 + 6 files changed, 58 insertions(+), 5 deletions(-) create mode 100644 sql/remake_insert_posts_and_insert_ids.sql diff --git a/sql/initialize.sql b/sql/initialize.sql index bf58149..341e0b6 100644 --- a/sql/initialize.sql +++ b/sql/initialize.sql @@ -72,7 +72,7 @@ CREATE TABLE IF NOT EXISTS posts , email text , body_search_index tsvector , thread_id bigint NOT NULL --- , TODO: embed + , embed text , CONSTRAINT unique_thread_board_id_constraint UNIQUE (thread_id, board_post_id) , CONSTRAINT thread_fk FOREIGN KEY (thread_id) REFERENCES threads (thread_id) ON DELETE CASCADE ); @@ -225,6 +225,7 @@ RETURNS TABLE ( body text, subject text, thread_id bigint, + embed text, board_thread_id bigint, pathpart text, site_name text, @@ -250,7 +251,8 @@ RETURNS TABLE ( top.bump_time, posts.body, posts.subject, - posts.thread_id + posts.thread_id, + posts.embed FROM top JOIN posts ON top.thread_id = posts.thread_id WHERE creation_time < max_time diff --git a/sql/remake_fetch_catalog.sql b/sql/remake_fetch_catalog.sql index bd3fecd..3c0f7ce 100644 --- a/sql/remake_fetch_catalog.sql +++ b/sql/remake_fetch_catalog.sql @@ -13,6 +13,7 @@ RETURNS TABLE ( body text, subject text, thread_id bigint, + embed text, board_thread_id bigint, pathpart text, site_name text, @@ -38,7 +39,8 @@ RETURNS TABLE ( top.bump_time, posts.body, posts.subject, - posts.thread_id + posts.thread_id, + posts.embed FROM top JOIN posts ON top.thread_id = posts.thread_id WHERE creation_time < max_time diff --git a/sql/remake_insert_posts_and_insert_ids.sql b/sql/remake_insert_posts_and_insert_ids.sql new file mode 100644 index 0000000..93db320 --- /dev/null +++ b/sql/remake_insert_posts_and_insert_ids.sql @@ -0,0 +1,44 @@ +REVOKE EXECUTE ON FUNCTION insert_posts_and_return_ids FROM PUBLIC; +DROP FUNCTION IF EXISTS insert_posts_and_return_ids(new_posts posts[]); + +CREATE OR REPLACE FUNCTION insert_posts_and_return_ids(new_posts posts[]) +RETURNS TABLE (post_id bigint, board_post_id bigint, thread_id bigint) AS $$ +WITH +selected AS ( + SELECT post_id, board_post_id, thread_id, embed + FROM posts + WHERE (thread_id, board_post_id) IN (SELECT thread_id, board_post_id FROM unnest(new_posts)) +), +to_update AS ( + SELECT s.post_id, np.embed + FROM unnest(new_posts) AS np + JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id + WHERE s.embed IS DISTINCT FROM np.embed +), +updated AS ( + UPDATE posts p + SET embed = tu.embed + FROM to_update tu + WHERE p.post_id = tu.post_id + RETURNING p.post_id, p.board_post_id, p.thread_id, p.embed +), +to_insert AS ( + SELECT np.* + FROM unnest(new_posts) AS np + LEFT OUTER JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id + WHERE s.post_id IS NULL +), +inserted AS ( + INSERT INTO posts (board_post_id, creation_time, body, subject, name, email, thread_id, embed) + SELECT board_post_id, creation_time, body, subject, name, email, thread_id, embed + FROM to_insert + RETURNING post_id, board_post_id, thread_id, embed +) +SELECT post_id, board_post_id, thread_id FROM inserted +UNION ALL +SELECT post_id, board_post_id, thread_id FROM updated +UNION ALL +SELECT post_id, board_post_id, thread_id FROM selected WHERE post_id NOT IN (SELECT post_id FROM updated); +$$ LANGUAGE sql; + +GRANT EXECUTE ON FUNCTION insert_posts_and_return_ids TO chan_archiver; diff --git a/src/Backfill.hs b/src/Backfill.hs index 866cc2e..1741633 100644 --- a/src/Backfill.hs +++ b/src/Backfill.hs @@ -212,6 +212,7 @@ apiPostToArchivePost thread post = , Posts.subject = JSONPosts.sub post , Posts.email = JSONPosts.email post , Posts.thread_id = Threads.thread_id thread + , Posts.embed = JSONPosts.embed post } -- | A version of 'concatMap' that works with a monadic predicate. @@ -586,7 +587,10 @@ processBackupDirectory settings = do -- - it's saged by a mod -- - the post has sage in the email field -- - the thread is full. - +-- +-- Better to support all those flags via the api: saged, locked, cyclical?, sticky +-- - deleted could be there too +-- - edited could be there too main :: IO () main = do diff --git a/src/Common b/src/Common index 3f420d3..a5ad727 160000 --- a/src/Common +++ b/src/Common @@ -1 +1 @@ -Subproject commit 3f420d3131eb0477b522103a582732a5ba365f19 +Subproject commit a5ad7272548d0096ef7aca3ac354d69616adbdd4 diff --git a/src/JSONPost.hs b/src/JSONPost.hs index 14c5296..510e9ea 100644 --- a/src/JSONPost.hs +++ b/src/JSONPost.hs @@ -22,6 +22,7 @@ data Post = Post , locked :: Maybe Int , cyclical :: Maybe J.Cyclical , last_modified :: Int + , embed :: Maybe Text , board :: Text , files :: Maybe [J.File] , resto :: Int