increase POST request timeout

- alternative post insert function to set the embed field on existing
  posts that don't have one

sql/initialize.sql

@@ -72,7 +72,7 @@ CREATE TABLE IF NOT EXISTS posts
     , email text
     , body_search_index tsvector
     , thread_id bigint NOT NULL
---  , TODO: embed
+    , embed text
     , CONSTRAINT unique_thread_board_id_constraint UNIQUE (thread_id, board_post_id)
     , CONSTRAINT thread_fk FOREIGN KEY (thread_id) REFERENCES threads (thread_id) ON DELETE CASCADE
     );
@@ -225,6 +225,7 @@ RETURNS TABLE (
     body text,
     subject text,
     thread_id bigint,
+    embed text,
     board_thread_id bigint,
     pathpart text,
     site_name text,
@@ -250,7 +251,8 @@ RETURNS TABLE (
                 top.bump_time,
                 posts.body,
                 posts.subject,
-                posts.thread_id
+                posts.thread_id,
+                posts.embed
             FROM top
             JOIN posts ON top.thread_id = posts.thread_id
             WHERE creation_time < max_time

sql/remake_fetch_catalog.sql

@@ -13,6 +13,7 @@ RETURNS TABLE (
     body text,
     subject text,
     thread_id bigint,
+    embed text,
     board_thread_id bigint,
     pathpart text,
     site_name text,
@@ -38,7 +39,8 @@ RETURNS TABLE (
                 top.bump_time,
                 posts.body,
                 posts.subject,
-                posts.thread_id
+                posts.thread_id,
+                posts.embed
             FROM top
             JOIN posts ON top.thread_id = posts.thread_id
             WHERE creation_time < max_time

sql/remake_insert_posts_and_insert_ids.sql

@@ -0,0 +1,44 @@
+REVOKE EXECUTE ON FUNCTION insert_posts_and_return_ids FROM PUBLIC;
+DROP FUNCTION IF EXISTS insert_posts_and_return_ids(new_posts posts[]);
+
+CREATE OR REPLACE FUNCTION insert_posts_and_return_ids(new_posts posts[])
+RETURNS TABLE (post_id bigint, board_post_id bigint, thread_id bigint) AS $$
+WITH 
+selected AS (
+    SELECT post_id, board_post_id, thread_id, embed
+    FROM posts
+    WHERE (thread_id, board_post_id) IN (SELECT thread_id, board_post_id FROM unnest(new_posts))
+),
+to_update AS (
+    SELECT s.post_id, np.embed
+    FROM unnest(new_posts) AS np
+    JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id
+    WHERE s.embed IS DISTINCT FROM np.embed
+),
+updated AS (
+    UPDATE posts p
+    SET embed = tu.embed
+    FROM to_update tu
+    WHERE p.post_id = tu.post_id
+    RETURNING p.post_id, p.board_post_id, p.thread_id, p.embed
+),
+to_insert AS (
+    SELECT np.*
+    FROM unnest(new_posts) AS np
+    LEFT OUTER JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id
+    WHERE s.post_id IS NULL
+),
+inserted AS (
+    INSERT INTO posts (board_post_id, creation_time, body, subject, name, email, thread_id, embed)
+    SELECT board_post_id, creation_time, body, subject, name, email, thread_id, embed
+    FROM to_insert
+    RETURNING post_id, board_post_id, thread_id, embed
+)
+SELECT post_id, board_post_id, thread_id FROM inserted
+UNION ALL
+SELECT post_id, board_post_id, thread_id FROM updated
+UNION ALL
+SELECT post_id, board_post_id, thread_id FROM selected WHERE post_id NOT IN (SELECT post_id FROM updated);
+$$ LANGUAGE sql;
+
+GRANT EXECUTE ON FUNCTION insert_posts_and_return_ids   TO chan_archiver;

src/Backfill.hs

@@ -212,6 +212,7 @@ apiPostToArchivePost thread post =
     , Posts.subject         = JSONPosts.sub post
     , Posts.email           = JSONPosts.email post
     , Posts.thread_id       = Threads.thread_id thread
+    , Posts.embed           = JSONPosts.embed post
     }
 
 -- | A version of 'concatMap' that works with a monadic predicate.
@@ -586,7 +587,10 @@ processBackupDirectory settings = do
 --          - it's saged by a mod
 --          - the post has sage in the email field
 --          - the thread is full.
-
+--
+-- Better to support all those flags via the api: saged, locked, cyclical?, sticky
+--      - deleted could be there too
+--      - edited could be there too
 
 main :: IO ()
 main = do

src/Common

@@ -1 +1 @@
-Subproject commit 3f420d3131eb0477b522103a582732a5ba365f19
+Subproject commit a5ad7272548d0096ef7aca3ac354d69616adbdd4

src/DataClient.hs

@@ -25,6 +25,7 @@ import Network.HTTP.Client
     ( newManager
     , managerSetMaxHeaderLength
     , httpLbs
+    , responseTimeoutNone
     )
 import qualified Data.ByteString.Lazy.Char8 as BL
 import Network.HTTP.Client.Conduit (defaultManagerSettings)
@@ -90,8 +91,9 @@ post
   -> IO (Either HttpError LBS.ByteString)
 post settings path payload return_repr = do
     let requestUrl = T.postgrest_url settings ++ path
-    initReq <- parseRequest requestUrl
+    req <- parseRequest requestUrl
-    let req = setRequestMethod "POST"
+    let initReq = setRequestResponseTimeout responseTimeoutNone req
+    let request = setRequestMethod "POST"
             . setRequestHeader "Authorization" [ jwt_header ]
             . setRequestHeader "Content-Type" [ "application/json" ]
             . setRequestBodyLBS payload
@@ -100,7 +102,7 @@ post settings path payload return_repr = do
 
     putStrLn $ "posting to " ++ requestUrl
     -- putStrLn $ "Payload: " ++ (LC8.unpack payload)
-    handleHttp (httpLBS req)
+    handleHttp (httpLBS request)
 
     where
       jwt_header = C8.pack $ "Bearer " ++ T.jwt settings

src/JSONPost.hs

@@ -22,6 +22,7 @@ data Post = Post
     , locked         :: Maybe Int
     , cyclical       :: Maybe J.Cyclical
     , last_modified  :: Int
+    , embed          :: Maybe Text
     , board          :: Text
     , files          :: Maybe [J.File]
     , resto          :: Int