function prepend(elem, children) { var child = elem.firstChild; if (child) { elem.insertBefore(children, child); } else { elem.appendChild(children); } } function _log() { for (var arg of arguments) { if (arg == null) { continue; } var pre = document.createElement('pre'); pre.appendChild(text(arg.toString())); document.body.appendChild(pre); try { prepend(document.body, pre); } catch (e) { var pre = document.createElement('pre'); pre.appendChild(text(e.toString())); document.body.appendChild(pre); } } } /* var console = { log: _log, error: _log }; */ var global_elements; var GlobalElemSelectors = [ 'style' , 'style.loading' , '.info' ] var GlobalVars = {}; var pageInfoText; var GlobalCancellableEvents = []; var cancel = GlobalCancellableEvents.push.bind(GlobalCancellableEvents); /** * Router */ // [ fpart : next ], content function RoutePart(fparts, content) { return { fparts, content } } function iter(iarray) { var i = 0; return function (f, g) { console.log(i, iarray.length, iarray); if (i < iarray.length) { console.log("calling", f, "with", iarray[i]); f(iarray[i++]); } else { console.log("calling", g); g(); } } } function resolvePath(routes, path, query_params) { console.log("resolvePath:", arguments); var next = iter(path); function goRoute(path_data, route_part, path_part) { var found = false; console.log("Route part:", route_part); console.log("Route part fparts:", route_part.fparts); for (var [fpart, next_route_part] of route_part.fparts) { var parsed_part = fpart(path_part); if (parsed_part) { console.log("parsed part:", parsed_part); path_data.push(parsed_part); next( goRoute.bind(this, path_data, next_route_part), resolve.bind(this, path_data, next_route_part) ); found = true; } } if (!found) { throw new Error("404 " + path_data.join('/') + '/' + path_part); } } function resolve(path_data, route_part) { console.log('calling content'); console.log(path_data, route_part, query_params); route_part.content(path_data, query_params); } var p = []; next(goRoute.bind(this, p, routes), resolve.bind(this, p, routes)); } function match(re) { return function (s) { if (re.test(s)) { return s; } } } var ROUTES = RoutePart( [ [ match(/^spam_post$/) , new RoutePart( [ [ match(/^[0-9]+$/) , new RoutePart([], showSpamPost) ] ] , null ) ] ] , handlePageRoot ); /** * Pagination */ function createPagination(start_page_num, url, name, page_size, renderContent) { const numDisplayedPages = 5; // Specify the number of numbered links to display // Helper function to update the URL with the new page number function updatePageNumberInURL(pageNum) { const url = new URL(window.location.href); const query_params = url.searchParams; console.log("updatePageNumberInURL", query_params.constructor.name); query_params.set(replaceAll(name, ' ', '_'), pageNum); console.log("Hello World 1"); changeUrl("/", query_params, true); } function createPaginationControls(page_num, dataset_size) { const container = document.createElement('nav'); container.classList.add('pagination'); // Calculate the total number of pages const total_pages = Math.ceil(dataset_size / page_size); console.log('page_num:', page_num, 'page_size:', page_size, 'dataset_size:', dataset_size, 'total_pages:', total_pages); function createNumberedLink(pageNum) { const link = document.createElement('a'); link.addEventListener('click', function(e) { //loadNthPage(url, pageNum - 1, page_size) e.preventDefault(); updatePageNumberInURL(pageNum - 1); }); link.appendChild(text(pageNum)); if (pageNum - 1 === page_num) { link.classList.add('active'); // Apply CSS class for active page } else { link.setAttribute('href', '#'); } return link; } // Add Previous link if applicable if (page_num > 0) { const prev_link = document.createElement('a'); prev_link.setAttribute('href', '#'); prev_link.addEventListener('click', function(e) { e.preventDefault(); //loadNthPage(url, page_num - 1, page_size) updatePageNumberInURL(page_num - 1); }); prev_link.appendChild(text('Previous')); container.appendChild(prev_link); } // Add numbered links const firstDisplayedPage = Math.max(1, page_num - Math.floor(numDisplayedPages / 2)); const lastDisplayedPage = Math.min(total_pages, firstDisplayedPage + numDisplayedPages - 1); if (firstDisplayedPage > 1) { container.appendChild(createNumberedLink(1)); if (firstDisplayedPage > 2) { container.appendChild(text('...')); } } for (let i = firstDisplayedPage; i <= lastDisplayedPage; i++) { container.appendChild(createNumberedLink(i)); } if (lastDisplayedPage < total_pages) { if (lastDisplayedPage < total_pages - 1) { container.appendChild(text('...')); } container.appendChild(createNumberedLink(total_pages)); } // Add Next link if applicable if (page_num < total_pages - 1) { const next_link = document.createElement('a'); next_link.setAttribute('href', '#'); next_link.addEventListener('click', function(e) { //loadNthPage(url, page_num + 1, page_size) e.preventDefault(); updatePageNumberInURL(page_num + 1); }); next_link.appendChild(text('Next')); container.appendChild(next_link); } return container; } function loadNthPage(url, page_num) { // Calculate the range of results based on the current page const start = page_num * page_size; const end = start + page_size - 1; var headers = { 'Authorization': 'Bearer ' + GlobalVars.settings.jwt, //'Prefer': 'count=estimated', 'Prefer': 'count=exact', 'Range-Unit': 'items', 'Range': `${start}-${end}` }; pageInfoText.innerText = `Loading page ${page_num} of ${name}`; cancelGlobalEvts(); return get(url, cancel, headers) .then(function(response) { // Retrieve the range and total number of results from the HTTP headers const rangeHeader = response.getResponseHeader('Content-Range'); const [start, end, total_results] = parseRangeHeader(rangeHeader); const json = JSON.parse(response.responseText); const controlsA = createPaginationControls(page_num, total_results); const controlsB = createPaginationControls(page_num, total_results); renderContent(json, controlsA, controlsB); pageInfoText.innerText = `${name} page ${page_num + 1}/${Math.ceil(total_results / page_size)}`; }) .catch(caught.bind(this, `Failed to load ${url} page ${page_num}. Reason:`)); } loadNthPage(url, start_page_num); } // Helper function to parse the Content-Range header function parseRangeHeader(rangeHeader) { const match = rangeHeader.match(/(\d+)-(\d+)\/(\d+)/); if (match) { const start = parseInt(match[1]); const end = parseInt(match[2]); const totalResults = parseInt(match[3]); return [start, end, totalResults]; } return [0, 0, 0]; // Default values if the header is not available or not in the expected format } /** * Functions */ function resolveResponse(resolve, reject) { return function(e) { const status = e.target.getResponseHeader("Status"); if (status != null) { const status_code = Number(status.split(" ")[0]); if (status_code >= 200 && status_code < 300) { resolve(e.target.responseText); } else { reject(new Error('API responded with ' + status + ". " + e.target.responseText)); } } else { if (e.target.status >= 200 && e.target.status < 300) { return resolve(e.target); } else { reject(new Error('API responded with ' + e.target.status + ". " + e.target.responseText)); } } } } function cancelGlobalEvts() { while (GlobalCancellableEvents.length) { GlobalCancellableEvents.pop().abort(); } } function get(url, xhr_req_follower, headers) { return new Promise(function (resolve, reject) { var req = new XMLHttpRequest(); xhr_req_follower(req); req.onload = resolveResponse(resolve, reject); req.onerror = reject; req.open('GET', url); if (headers) { Object.entries(headers) .forEach(function([header_name, header_value]) { req.setRequestHeader(header_name, header_value); }); } req.send(); }); } function post(url, body, xhr_req_follower, headers) { return new Promise(function (resolve, reject) { var req = new XMLHttpRequest(); xhr_req_follower(req); req.onload = resolveResponse(resolve, reject); req.onerror = reject; req.open('POST', url); if (headers) { Object.entries(headers) .forEach(function([header_name, header_value]) { req.setRequestHeader(header_name, header_value); }); } req.setRequestHeader('Content-Type', 'application/json'); if (body) { req.send(JSON.stringify(body)); } else { req.send(); } }); } // Given a BigInt that is packed bytes, unpack the bytes into an ArrayBuffer function unpackBytes(bigint) { const buffer = new ArrayBuffer(8); const view = new DataView(buffer); view.setBigInt64(0, bigint); return buffer; } function bufferToHex(buffer) { // Create a Uint8Array view to access the bytes const uint8Array = new Uint8Array(buffer); // Convert each byte to its hexadecimal representation const hexArray = Array.from(uint8Array, byte => byte.toString(16).padStart(2, '0')); // Join the hexadecimal values into a single string return hexArray.join(''); } function hammingDistance(buffer1, buffer2) { if (buffer1.byteLength !== buffer2.byteLength) { throw new Error("Buffer lengths must be equal"); } const view1 = new DataView(buffer1); const view2 = new DataView(buffer2); let distance = 0; for (let i = 0; i < buffer1.byteLength; i++) { const byte1 = view1.getUint8(i); const byte2 = view2.getUint8(i); let xor = byte1 ^ byte2; while (xor !== 0) { distance++; xor &= xor - 1; } } return distance; } function text(s) { return document.createTextNode(s); } function _mkelem(etype, child) { var elem = document.createElement(etype); if (child) { elem.appendChild(child); } return elem; } var h3 = _mkelem.bind(this, 'h3'); var h4 = _mkelem.bind(this, 'h4'); var div = _mkelem.bind(this, 'div'); var li = _mkelem.bind(this, 'li'); var span = _mkelem.bind(this, 'span'); var italic = _mkelem.bind(this, 'i'); var bold = _mkelem.bind(this, 'b'); var ul = _mkelem.bind(this, 'ul'); function keepElements(elem_selectors) { var document_root = document.querySelector('main'); document_root.innerHTML = ''; for (var i=0; i < elem_selectors.length; i++) { var sel = elem_selectors[i]; var elem = global_elements[sel]; document_root.appendChild(elem); } } function caught(m, e) { console.error(m); console.error(e); for (var property in e) { console.error(property + ": " + e[property]); } alert(new Error(m + e.message)); }; function mkSpamImgElement(img_description) { var img_elem = document.createElement('img'); var mime = img_description.mimetype; var thumb_url; var illegal = img_description.illegal; console.log('img_description.illegal', img_description.illegal); if (illegal) { thumb_url = `/bin/?path=${GlobalVars.settings.redacted_thumbnail_path}&mimetype=image/jpeg` } else if (mime == "audio/mpeg") { thumb_url = `/bin/?path=${GlobalVars.settings.audio_mpeg_thumbnail_path}&mimetype=image/jpeg` } else { thumb_url = `/bin/?path=${GlobalVars.settings.content_directory}/${img_description.md5_hash}_thumbnail.jpg&mimetype=image/jpeg` } img_elem.setAttribute('src', thumb_url); img_elem.setAttribute('decoding', 'async'); img_elem.setAttribute('loading', 'lazy'); var a_elem; if (!illegal) { var image_url = `/bin/?path=${GlobalVars.settings.content_directory}/${img_description.md5_hash}.attachment&mimetype=${mime}` a_elem = document.createElement('a'); a_elem.setAttribute('href', image_url); a_elem.appendChild(img_elem); } else { a_elem = div(img_elem); } var img_container_elem = div(a_elem); img_container_elem.classList.add('post--image_container'); return img_container_elem; } function renderNav() { var nav = document.createElement('nav'); var a_elem = _mkelem('a', text('All Known Spam Posts')); a_elem.addEventListener('click', function(e) { e.preventDefault(); changeUrl('/', new URLSearchParams(), true); }); a_elem.setAttribute('href', '#'); nav.appendChild(a_elem); return nav } function linkToBoard(website_name, board_name, thread_id) { let root = `${GlobalVars.settings.website_urls[website_name]}/${board_name}` console.log(GlobalVars.settings.website_urls, GlobalVars.settings.website_urls[website_name], board_name); if (thread_id) { return root + `/res/${thread_id}.html`; } else { return root + 'index.html'; } } function attachmentCanBeMarkedIllegal(attachment) { return attachment.phash != null && !attachment.illegal; } function shouldDisplayMarkIllegalButton(post) { let result = false; for (let attachment of (post.known_spam_post_attachment || [])) { result = result || attachmentCanBeMarkedIllegal(attachment); } return result; } function isIllegal(post) { let result = false; for (let attachment of (post.known_spam_post_attachment || [])) { result = result || attachment.illegal; } return result; } async function onClickMarkIllegal(phashes, e) { console.log('CLICK', phashes); e.stopPropagation(); e.preventDefault(); const url = new URL('/illegal', window.location.origin); phashes.forEach(function(p) { console.log('onClickMarkIllegal', typeof p, p); url.searchParams.append('phash', p); }); await post(url.toString(), null, cancel) .then(function() { location.reload(); }) .catch(caught.bind(this, "Failed to mark post illegal because:")); } function renderPostElem(post) { const postContainer = div(); postContainer.classList.add('post'); // Header with timestamp, link to the thread const postHeader = div(); postHeader.classList.add('post--header'); postContainer.appendChild(postHeader); console.log('renderPostElem POST:', post); if (post.website_name != null) { const link_url = linkToBoard(post.website_name, post.board_name, post.thread_id); const boardlink = span() const boardlink_a = document.createElement('a'); boardlink_a.appendChild(text(post.website_name)); boardlink_a.setAttribute('href', link_url); boardlink_a.setAttribute('title', 'Destination this post was originally headed for (thread or board)'); boardlink.appendChild(text('[')); boardlink.appendChild(boardlink_a); boardlink.appendChild(text(']')); postHeader.appendChild(boardlink); } postHeader.appendChild(span(text((new Date(post.time_stamp).toUTCString())))); if (shouldDisplayMarkIllegalButton(post)) { const mark_illegal = span() mark_illegal.classList.add('post--mark_illegal'); const mark_illegal_a = document.createElement('a'); mark_illegal_a.appendChild(text('Mark Illegal')); mark_illegal_a.setAttribute('href', '#'); mark_illegal_a.setAttribute('title', 'Permanently delete associated pictures and thumbnails but keep metadata for future matching'); mark_illegal.appendChild(text('[')); mark_illegal.appendChild(mark_illegal_a); mark_illegal.appendChild(text(']')); const phashes = (post.known_spam_post_attachment || []) .map(function(p) { return bufferToHex(unpackBytes(BigInt(p.phash))); }); mark_illegal_a.addEventListener('click', onClickMarkIllegal.bind(this, phashes)); postHeader.appendChild(mark_illegal); } else if (isIllegal(post)) { const mark_illegal = span(text('illegal')) mark_illegal.classList.add('post--is_illegal'); postHeader.appendChild(mark_illegal); } postContainer.appendChild(postHeader); const postElem = div(); postElem.classList.add('post--body_container'); postContainer.appendChild(postElem); const bodyTextElem = div(); bodyTextElem.classList.add('post--body'); bodyTextElem.innerText = post.body; if (post.known_spam_post_attachment) { post.known_spam_post_attachment.forEach(function(attachment) { postElem.appendChild(mkSpamImgElement(attachment)); }); } postElem.appendChild(bodyTextElem); return postContainer; } function renderKnownSpamAttachments(attachments) { var container = div(); container.classList.add('attachment'); attachments.forEach(function(a) { container.appendChild(mkSpamImgElement(a)); }); return container; } function renderThings(things_map) { var container = div(); for (const [key, value] of Object.entries(things_map)) { var entry = div(); entry.appendChild(span(bold(text(key + ": ")))); entry.appendChild(span(text(value))); container.appendChild(entry); } return container; } function renderAttachmentReason(attachment, known_spam_post_attachments) { let container = div(); container.classList.add('attachment-reason'); container.appendChild(mkSpamImgElement(attachment)); let phash_buffer = unpackBytes(BigInt(attachment['phash'])); let distance = (64 - Math.min(...known_spam_post_attachments.map(function(known_attachment) { let known_buffer = unpackBytes(BigInt(known_attachment['phash'])); let distance = hammingDistance(known_buffer, phash_buffer); return distance; }))) / 64; let things = { 'mimetype': attachment['mimetype'], 'time stamp': attachment['time_stamp'], 'md5': attachment['md5_hash'], 'phash (hex)': bufferToHex(phash_buffer), 'match': `${Math.round(distance * 100 * 10) / 10}%` }; container.appendChild(renderThings(things)); return container; } function renderReasons(post) { var container = div(h3(text('Reasons for being considered spam:'))); container.appendChild(renderReasonsUl(post)); var reasonDetails = post.reason_details; if (reasonDetails) { var reasonDetailsContainer = div(); reasonDetailsContainer.className = 'reason-details'; if (reasonDetails.frequent_attachment_details) { reasonDetailsContainer.appendChild(h4(text('Previously posted matching attachments:'))); reasonDetails.frequent_attachment_details.forEach(function (attachment) { reasonDetailsContainer.appendChild( renderAttachmentReason(attachment, post.known_spam_post_attachment) ); }); } if (reasonDetails.frequent_text_details) { var textPosts = div(); reasonDetails.frequent_text_details.forEach(function(post) { textPosts.appendChild(renderPostElem(post)); }); reasonDetailsContainer.appendChild(h4(text('Previously posted matching text:'))); reasonDetailsContainer.appendChild(textPosts); } container.appendChild(h3(text('Reason Details:'))); container.appendChild(reasonDetailsContainer); } return container; } var REASONS = { RECENTLY_POSTED_ATTACHMENT_ABOVE_RATE: 1 << 0, LEXICAL_ANALYSIS_SHOWS_NONSENSE: 1 << 1, SEGMENTED_AVG_ENTROPY_TOO_HIGH: 1 << 2, TEXT_MATCHES_KNOWN_SPAM: 1 << 3, RECENTLY_POSTED_TEXT_ABOVE_RATE: 1 << 4, TEXT_IS_URL: 1 << 5, TEXT_IS_ONE_LONG_WORD: 1 << 6, TEXT_IS_RANDOM_WORDS: 1 << 7 }; function reasonsFromBitmap(n) { var reasons = new Set(); for (const [reason, bit] of Object.entries(REASONS)) { if (n & bit) { reasons.add(reason); } } return reasons; } function renderReasonsUl(post){ var reasons = ul(); reasonsFromBitmap(post.reason).forEach(function(r) { reasons.appendChild(li(text(r))); }); return reasons; } function renderOverviewPost(post) { var postContainer = renderPostElem(post); postContainer.appendChild(h3(text('Reasons:'))); postContainer.appendChild(div(renderReasonsUl(post))); postContainer.classList.add('post--overview'); return postContainer; } function replaceAll(input, find, replace) { var regex = new RegExp(find, "g"); return input.replace(regex, replace); } function loadKnownSpamPosts(params) { console.log("loadKnownSpamPosts"); const pageName = "known spam posts"; console.log("HELLO WORLD 1"); var url = GlobalVars.settings.postgrest_url + '/known_spam_post?select=*,known_spam_post_attachment(*,phash::text)' const pageNum = Number(params.get(replaceAll(pageName, ' ', '_')) || 0); function renderContent(json, controlsA, controlsB) { var mainElem = document.querySelector('main'); mainElem.innerHTML = ''; mainElem.appendChild(controlsA); json.forEach(function (post) { var postElem = renderOverviewPost(post); postElem.addEventListener('click', changeUrl.bind( this, '/spam_post/' + post.text_post_id, new URLSearchParams(), true )); mainElem.appendChild(postElem); }); mainElem.appendChild(controlsB); } createPagination(pageNum, url, pageName, 25, renderContent); } function changeUrl(path, search_query, push_state) { console.log('changeURL'); var event = new CustomEvent( 'urlchange', { detail: { path: path, query: search_query, push_state: push_state, } }); window.dispatchEvent(event); } function handlePageRoot(_, query_params) { console.log("handlePageRoot", JSON.stringify(query_params)); return loadKnownSpamPosts(query_params); } function showSpamPost(path_data, _) { console.log(path_data); var post_id = path_data[1]; pageInfoText.innerText = 'Loading known spam post ' + post_id; var url = GlobalVars.settings.postgrest_url + '/known_spam_post?text_post_id=eq.' + post_id + '&select=*,known_spam_post_attachment(*,phash::text)'; // Have to load the phash as text here because javascripts json parser won't use BigInt when appropriate because javascript Numbers are based on floats and we're dealing with a 64bit signed int. var headers = { 'Authorization': 'Bearer ' + GlobalVars.settings.jwt }; var postSectionElem = document.createElement('section'); var mainElem = document.querySelector('main'); mainElem.innerHTML = ''; mainElem.appendChild(renderNav()); mainElem.appendChild(postSectionElem); var h2 = document.createElement('h2'); h2.innerText = 'Known Spam Attachments'; mainElem.appendChild(postSectionElem); mainElem.appendChild(h2); var knownSpamAttachmentsSectionElem = document.createElement('section'); mainElem.append(knownSpamAttachmentsSectionElem); var reasonsSectionElem = document.createElement('section'); mainElem.appendChild(reasonsSectionElem); get(url, cancel, headers) .then(function(response) { var json = JSON.parse(response.responseText); console.log(json); pageInfoText.innerText = 'Known Spam Post'; json.forEach(function(p) { var postElem = renderPostElem(p); postSectionElem.appendChild(postElem); reasonsSectionElem.appendChild(renderReasons(p)) }); }) .catch(caught.bind(this, "Failed to load known spam post. Reason:")); url = GlobalVars.settings.postgrest_url + '/known_spam_attachments?post_id=eq.' + post_id; get(url, cancel, headers) .then(function(response) { var json = JSON.parse(response.responseText); console.log("known spam attachments:", json); knownSpamAttachmentsSectionElem .appendChild(renderKnownSpamAttachments(json)); }) .catch(caught.bind(this, "Failed to load known spam attachments. Reason:")); } function onUrlChange(e) { console.log('Hello World 2'); var path = e.detail.path.split('/').filter(function(s){ return s.length; }); console.log(e, e.detail, path); if (e.detail.push_state) { var search_query = e.detail.query; search_query.set('__path', e.detail.path); console.log("onUrlChange push_state search_query:", search_query, typeof search_query); window.history.pushState(null, "", window.location.origin + window.location.pathname + '?' + search_query.toString()); } resolvePath(ROUTES, path, e.detail.query); } function aClickNav(e) { e.preventDefault(); e.stopPropagation(); console.log(this, this.href); window.history.pushState(null, "", this.href); onRealUrlChange(); } function bindEventHandlers() { window.addEventListener('urlchange', onUrlChange); window.addEventListener('popstate', onRealUrlChange); } function gatherElements() { global_elements = {}; var qs; for (var i=0; i < GlobalElemSelectors.length; i++) { var sel = GlobalElemSelectors[i]; var elem = document.querySelector(sel); global_elements[sel] = elem; } } function onRealUrlChange() { const url = new URL(window.location.href); const query_params = url.searchParams; const path = query_params.get('__path') || '/'; changeUrl(path, query_params); } function initPageInfoText() { pageInfoText = document.createElement('div'); pageInfoText.className = 'info'; prepend(document.querySelector('.info-container'), pageInfoText); } function getSettings() { cancelGlobalEvts(); pageInfoText.innerText = 'Loading settings'; return get('/static/settings.json', cancel) .then(function(response) { var result = JSON.parse(response.responseText); console.log("settings response:", result); GlobalVars.settings = result; pageInfoText.innerText = 'Have settings.'; }) .catch(caught.bind(this, "Failed to load JWT: ")); } /** * Init */ initPageInfoText(); // gatherElements(); bindEventHandlers(); // searchInputHandler(); getSettings() .then(onRealUrlChange);