sql/initialize.sql

@@ -72,7 +72,7 @@ CREATE TABLE IF NOT EXISTS posts
     , email text
     , body_search_index tsvector
     , thread_id bigint NOT NULL
-    , embed text
+--  , TODO: embed
     , CONSTRAINT unique_thread_board_id_constraint UNIQUE (thread_id, board_post_id)
     , CONSTRAINT thread_fk FOREIGN KEY (thread_id) REFERENCES threads (thread_id) ON DELETE CASCADE
     );
@@ -225,7 +225,6 @@ RETURNS TABLE (
     body text,
     subject text,
     thread_id bigint,
-    embed text,
     board_thread_id bigint,
     pathpart text,
     site_name text,
@@ -251,8 +250,7 @@ RETURNS TABLE (
                 top.bump_time,
                 posts.body,
                 posts.subject,
-                posts.thread_id,
-                posts.embed
+                posts.thread_id
             FROM top
             JOIN posts ON top.thread_id = posts.thread_id
             WHERE creation_time < max_time

sql/remake_fetch_catalog.sql

@@ -13,7 +13,6 @@ RETURNS TABLE (
     body text,
     subject text,
     thread_id bigint,
-    embed text,
     board_thread_id bigint,
     pathpart text,
     site_name text,
@@ -39,8 +38,7 @@ RETURNS TABLE (
                 top.bump_time,
                 posts.body,
                 posts.subject,
-                posts.thread_id,
-                posts.embed
+                posts.thread_id
             FROM top
             JOIN posts ON top.thread_id = posts.thread_id
             WHERE creation_time < max_time

sql/remake_insert_posts_and_insert_ids.sql

@@ -1,44 +0,0 @@
-REVOKE EXECUTE ON FUNCTION insert_posts_and_return_ids FROM PUBLIC;
-DROP FUNCTION IF EXISTS insert_posts_and_return_ids(new_posts posts[]);
-
-CREATE OR REPLACE FUNCTION insert_posts_and_return_ids(new_posts posts[])
-RETURNS TABLE (post_id bigint, board_post_id bigint, thread_id bigint) AS $$
-WITH 
-selected AS (
-    SELECT post_id, board_post_id, thread_id, embed
-    FROM posts
-    WHERE (thread_id, board_post_id) IN (SELECT thread_id, board_post_id FROM unnest(new_posts))
-),
-to_update AS (
-    SELECT s.post_id, np.embed
-    FROM unnest(new_posts) AS np
-    JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id
-    WHERE s.embed IS DISTINCT FROM np.embed
-),
-updated AS (
-    UPDATE posts p
-    SET embed = tu.embed
-    FROM to_update tu
-    WHERE p.post_id = tu.post_id
-    RETURNING p.post_id, p.board_post_id, p.thread_id, p.embed
-),
-to_insert AS (
-    SELECT np.*
-    FROM unnest(new_posts) AS np
-    LEFT OUTER JOIN selected s ON np.thread_id = s.thread_id AND np.board_post_id = s.board_post_id
-    WHERE s.post_id IS NULL
-),
-inserted AS (
-    INSERT INTO posts (board_post_id, creation_time, body, subject, name, email, thread_id, embed)
-    SELECT board_post_id, creation_time, body, subject, name, email, thread_id, embed
-    FROM to_insert
-    RETURNING post_id, board_post_id, thread_id, embed
-)
-SELECT post_id, board_post_id, thread_id FROM inserted
-UNION ALL
-SELECT post_id, board_post_id, thread_id FROM updated
-UNION ALL
-SELECT post_id, board_post_id, thread_id FROM selected WHERE post_id NOT IN (SELECT post_id FROM updated);
-$$ LANGUAGE sql;
-
-GRANT EXECUTE ON FUNCTION insert_posts_and_return_ids   TO chan_archiver;

src/Backfill.hs

@@ -212,7 +212,6 @@ apiPostToArchivePost thread post =
     , Posts.subject         = JSONPosts.sub post
     , Posts.email           = JSONPosts.email post
     , Posts.thread_id       = Threads.thread_id thread
-    , Posts.embed           = JSONPosts.embed post
     }
 
 -- | A version of 'concatMap' that works with a monadic predicate.
@@ -587,10 +586,7 @@ processBackupDirectory settings = do
 --          - it's saged by a mod
 --          - the post has sage in the email field
 --          - the thread is full.
---
--- Better to support all those flags via the api: saged, locked, cyclical?, sticky
---      - deleted could be there too
---      - edited could be there too
+
 
 main :: IO ()
 main = do

src/Common

@@ -1 +1 @@
-Subproject commit a5ad7272548d0096ef7aca3ac354d69616adbdd4
+Subproject commit 3f420d3131eb0477b522103a582732a5ba365f19

src/DataClient.hs

@@ -25,7 +25,6 @@ import Network.HTTP.Client
     ( newManager
     , managerSetMaxHeaderLength
     , httpLbs
-    , responseTimeoutNone
     )
 import qualified Data.ByteString.Lazy.Char8 as BL
 import Network.HTTP.Client.Conduit (defaultManagerSettings)
@@ -91,9 +90,8 @@ post
   -> IO (Either HttpError LBS.ByteString)
 post settings path payload return_repr = do
     let requestUrl = T.postgrest_url settings ++ path
-    req <- parseRequest requestUrl
-    let initReq = setRequestResponseTimeout responseTimeoutNone req
-    let request = setRequestMethod "POST"
+    initReq <- parseRequest requestUrl
+    let req = setRequestMethod "POST"
             . setRequestHeader "Authorization" [ jwt_header ]
             . setRequestHeader "Content-Type" [ "application/json" ]
             . setRequestBodyLBS payload
@@ -102,7 +100,7 @@ post settings path payload return_repr = do
 
     putStrLn $ "posting to " ++ requestUrl
     -- putStrLn $ "Payload: " ++ (LC8.unpack payload)
-    handleHttp (httpLBS request)
+    handleHttp (httpLBS req)
 
     where
       jwt_header = C8.pack $ "Bearer " ++ T.jwt settings

src/JSONPost.hs

@@ -22,7 +22,6 @@ data Post = Post
     , locked         :: Maybe Int
     , cyclical       :: Maybe J.Cyclical
     , last_modified  :: Int
-    , embed          :: Maybe Text
     , board          :: Text
     , files          :: Maybe [J.File]
     , resto          :: Int