function prepend(elem, children) { var child = elem.firstChild; if (child) { elem.insertBefore(children, child); } else { elem.appendChild(children); } } function _log() { for (var arg of arguments) { var pre = document.createElement('pre'); pre.appendChild(text(arg.toString())); document.body.appendChild(pre); try { prepend(document.body, pre); } catch (e) { var pre = document.createElement('pre'); pre.appendChild(text(e.toString())); document.body.appendChild(pre); } } } /* var console = { log: _log, error: _log }; */ var CurrentScore = null; var global_elements; var GlobalElemSelectors = [ 'style' , 'style.loading' , '.info' ] var GlobalVars = {}; var pageInfoText; var GlobalCancellableEvents = []; var cancel = GlobalCancellableEvents.push.bind(GlobalCancellableEvents); /** * Router */ // [ fpart : next ], content function RoutePart(fparts, content) { return { fparts, content } } function iter(iarray) { var i = 0; return function (f, g) { console.log(i, iarray.length, iarray); if (i < iarray.length) { console.log("calling", f, "with", iarray[i]); f(iarray[i++]); } else { console.log("calling", g); g(); } } } function resolvePath(routes, path, query_params) { console.log("resolvePath:", arguments); var next = iter(path); function goRoute(path_data, route_part, path_part) { var found = false; console.log("Route part:", route_part); console.log("Route part fparts:", route_part.fparts); for (var [fpart, next_route_part] of route_part.fparts) { var parsed_part = fpart(path_part); if (parsed_part) { console.log("parsed part:", parsed_part); path_data.push(parsed_part); next( goRoute.bind(this, path_data, next_route_part), resolve.bind(this, path_data, next_route_part) ); found = true; } } if (!found) { throw new Error("404 " + path_data.join('/') + '/' + path_part); } } function resolve(path_data, route_part) { console.log('calling content'); console.log(path_data, route_part); route_part.content(path_data, query_params); } var p = []; next(goRoute.bind(this, p, routes), resolve.bind(this, p, routes)); } function match(re) { return function (s) { if (re.test(s)) { return s; } } } var ROUTES = RoutePart( [ [ match(/^spam_post$/) , new RoutePart( [ [ match(/^[0-9]+$/) , new RoutePart([], showSpamPost) ] ] , null ) ] ] , handlePageRoot ); /** * Functions */ function resolveResponse(resolve, reject) { return function(e) { var status = e.target.getResponseHeader("Status"); if (status != null) { var status_code = Number(status.split(" ")[0]); if (status_code >= 200 && status_code < 300) { resolve(e.target.responseText); } else { reject(new Error('API responded with ' + status)); } } else { if (e.target.status >= 200 && e.target.status < 300) { return resolve(e.target.responseText); } else { reject(new Error('API responded with ' + e.target.status)); } } } } function cancelGlobalEvts() { while (GlobalCancellableEvents.length) { GlobalCancellableEvents.pop().abort(); } } function get(url, xhr_req_follower, headers) { return new Promise(function (resolve, reject) { var req = new XMLHttpRequest(); xhr_req_follower(req); req.onload = resolveResponse(resolve, reject); req.onerror = reject; req.open('GET', url); if (headers) { Object.entries(headers) .forEach(function([header_name, header_value]) { req.setRequestHeader(header_name, header_value); }); } req.send(); }); } // Given a BigInt that is packed bytes, unpack the bytes into an ArrayBuffer function unpackBytes(bigint) { const buffer = new ArrayBuffer(8); const view = new DataView(buffer); view.setBigInt64(0, bigint); return buffer; } function hexArrayFromBuffer(buffer) { // Create a Uint8Array view to access the bytes const uint8Array = new Uint8Array(buffer); // Convert each byte to its hexadecimal representation const hexArray = Array.from(uint8Array, byte => byte.toString(16).padStart(2, '0')); return uint8Array; } function bufferToHex(buffer) { // Join the hexadecimal values into a single string return hexArray(buffer).join(''); } function hammingDistance(buffer1, buffer2) { if (buffer1.byteLength !== buffer2.byteLength) { throw new Error("Buffer lengths must be equal"); } const view1 = new DataView(buffer1); const view2 = new DataView(buffer2); let distance = 0; for (let i = 0; i < buffer1.byteLength; i++) { const byte1 = view1.getUint8(i); const byte2 = view2.getUint8(i); let xor = byte1 ^ byte2; while (xor !== 0) { distance++; xor &= xor - 1; } } return distance; } function parseFilePath(s) { return JSON.parse(s); } function FileSize(s) { this.filesize = s; } function tree(rows) { var c = {}; for (var row of rows) { var path = parseFilePath(row.path); var tree = c; var previous = c; var filename = 'NOPATH'; for (var p of path) { previous = tree; if (p in tree) { tree = tree[p]; } else { var newtree = {}; tree[p] = newtree; tree = newtree; } filename = p; } previous[filename] = new FileSize(row.filesize); } console.log(c); return c; } function renderTree(rows) { var list_elem = ul(); function draw(list_elem, t) { Object.entries(t) .forEach(function([key, val]) { if (val instanceof FileSize) { var contents = div(span(text(key))); contents.appendChild(italic(text(val.filesize))); list_elem.appendChild(contents); } else { var new_list = ul(); var elem = li(text(key)); elem.appendChild(new_list); list_elem.appendChild(elem); draw(new_list, val); } }); } draw(list_elem, tree(rows)); return list_elem; } function assembleTable(rows) { var t = document.createElement('table'); rows.forEach(function(r) { var row = t.insertRow(); TableHeadings.forEach(function(header) { var cell = row.insertCell(); cell.appendChild(header[1](r)); }); }); var thead = t.createTHead().insertRow(); TableHeadingNames.forEach(function(header) { var th = document.createElement('th'); th.appendChild(text(header)); thead.appendChild(th); }); var countElem = div(text(rows.length + ' rows')) var wrapper = div(); wrapper.appendChild(t); wrapper.appendChild(countElem); return wrapper; } function text(s) { return document.createTextNode(s); } function _mkelem(etype, child) { var elem = document.createElement(etype); if (child) { elem.appendChild(child); } return elem; } var h3 = _mkelem.bind(this, 'h3'); var h4 = _mkelem.bind(this, 'h4'); var div = _mkelem.bind(this, 'div'); var li = _mkelem.bind(this, 'li'); var span = _mkelem.bind(this, 'span'); var italic = _mkelem.bind(this, 'i'); var bold = _mkelem.bind(this, 'b'); var ul = _mkelem.bind(this, 'ul'); function keepElements(elem_selectors) { var document_root = document.querySelector('main'); document_root.innerHTML = ''; for (var i=0; i < elem_selectors.length; i++) { var sel = elem_selectors[i]; var elem = global_elements[sel]; document_root.appendChild(elem); } } function loadJwt() { cancelGlobalEvts(); } function caught(m, e) { console.error(m); console.error(e); for (var property in e) { console.error(property + ": " + e[property]); } alert(new Error(m + e.message)); }; function mkSpamImgElement(img_description) { var img_elem = document.createElement('img'); var mime = img_description.mimetype; var thumb_url; if (mime == "audio/mpeg") { thumb_url = `/bin/?path=${GlobalVars.settings.audio_mpeg_thumbnail_path}&mimetype=image/jpeg` } else { thumb_url = `/bin/?path=${GlobalVars.settings.content_directory}/${img_description.md5_hash}_thumbnail.jpg&mimetype=image/jpeg` } var image_url = `/bin/?path=${GlobalVars.settings.content_directory}/${img_description.md5_hash}.attachment&mimetype=${mime}` img_elem.setAttribute('src', thumb_url); img_elem.setAttribute('decoding', 'async'); img_elem.setAttribute('loading', 'lazy'); var a_elem = document.createElement('a'); a_elem.setAttribute('href', image_url); a_elem.appendChild(img_elem); var img_container_elem = div(a_elem); img_container_elem.classList.add('post--image_container'); return img_container_elem; } function renderNav() { var nav = document.createElement('nav'); var a_elem = _mkelem('a', text('All Known Spam Posts')); a_elem.addEventListener('click', function() { changeUrl('/', null, true); }); a_elem.setAttribute('href', '#'); nav.appendChild(a_elem); return nav } function renderPostElem(post) { var postContainer = div(); postContainer.classList.add('post'); var postElem = div(); postElem.classList.add('post--body_container'); postContainer.appendChild(postElem); var bodyTextElem = div(); bodyTextElem.classList.add('post--body'); bodyTextElem.innerText = post.body; if (post.known_spam_post_attachment) { post.known_spam_post_attachment.forEach(function(attachment) { postElem.appendChild(mkSpamImgElement(attachment)); }); } postElem.appendChild(bodyTextElem); return postContainer; } function renderKnownSpamAttachments(attachments) { var container = div(); container.classList.add('attachment'); attachments.forEach(function(a) { container.appendChild(mkSpamImgElement(a)); }); return container; } function renderThings(things_map) { var container = div(); for (const [key, value] of Object.entries(things_map)) { var entry = div(); entry.appendChild(span(bold(text(key + ": ")))); entry.appendChild(span(text(value))); container.appendChild(entry); } return container; } function renderAttachmentReason(attachment) { var container = div(); container.classList.add('attachment-reason'); container.appendChild(mkSpamImgElement(attachment)); var things = { 'mimetype': attachment['mimetype'], 'time stamp': attachment['time_stamp'], 'md5': attachment['md5_hash'], 'phash raw': BigInt(attachment['phash']).toString(2), 'phash (hex)': hexArrayFromBuffer(unpackBytes(BigInt(attachment['phash']))), //'distance': hammingDistance(unpackBytes(BigInt(attachment['phash'])), unpackBytes(BigInt(attachment['phash']))), 'distance': hammingDistance(unpackBytes(BigInt(attachment['phash'])), unpackBytes(BigInt(0))), }; container.appendChild(renderThings(things)); return container; } function renderReasons(post) { var container = div(h3(text('Reasons for being considered spam:'))); container.appendChild(renderReasonsUl(post)); var reasonDetails = post.reason_details; if (reasonDetails) { var reasonDetailsContainer = div(); reasonDetailsContainer.className = 'reason-details'; if (reasonDetails.frequent_attachment_details) { reasonDetailsContainer.appendChild(h4(text('Previously posted matching attachments:'))); reasonDetails.frequent_attachment_details.forEach(function (attachment) { reasonDetailsContainer.appendChild(renderAttachmentReason(attachment)); }); } if (reasonDetails.frequent_text_details) { var textPosts = div(); reasonDetails.frequent_text_details.forEach(function(post) { textPosts.appendChild(renderPostElem(post)); }); reasonDetailsContainer.appendChild(h4(text('Previously posted matching text:'))); reasonDetailsContainer.appendChild(textPosts); } container.appendChild(h3(text('Reason Details:'))); container.appendChild(reasonDetailsContainer); } return container; } var REASONS = { RECENTLY_POSTED_ATTACHMENT_ABOVE_RATE: 1 << 0, LEXICAL_ANALYSIS_SHOWS_NONSENSE: 1 << 1, SEGMENTED_AVG_ENTROPY_TOO_HIGH: 1 << 2, TEXT_MATCHES_KNOWN_SPAM: 1 << 3, RECENTLY_POSTED_TEXT_ABOVE_RATE: 1 << 4, TEXT_IS_URL: 1 << 5, TEXT_IS_ONE_LONG_WORD: 1 << 6, TEXT_IS_RANDOM_WORDS: 1 << 7 }; function reasonsFromBitmap(n) { var reasons = new Set(); for (const [reason, bit] of Object.entries(REASONS)) { if (n & bit) { reasons.add(reason); } } return reasons; } function renderReasonsUl(post){ var reasons = ul(); reasonsFromBitmap(post.reason).forEach(function(r) { reasons.appendChild(li(text(r))); }); return reasons; } function renderOverviewPost(post) { var postContainer = renderPostElem(post); postContainer.appendChild(h3(text('Reasons:'))); postContainer.appendChild(div(renderReasonsUl(post))); postContainer.classList.add('post--overview'); return postContainer; } function loadKnownSpamPosts() { pageInfoText.innerText = 'Loading all known spam posts...'; cancelGlobalEvts(); var url = GlobalVars.settings.postgrest_url + '/known_spam_post?select=*,known_spam_post_attachment(*)' var headers = { 'Authorization': 'Bearer ' + GlobalVars.settings.jwt }; get(url, cancel, headers) .then(function(response) { var json = JSON.parse(response); console.log("results for", url, response); pageInfoText.innerText = 'Have these known spam posts:'; var mainElem = document.querySelector('main'); mainElem.innerHTML = ''; json.forEach(function (post) { var postElem = renderOverviewPost(post); postElem.addEventListener('click', function() { changeUrl('/spam_post/' + post.text_post_id, null, true); }); mainElem.appendChild(postElem); }); }) .catch(caught.bind(this, "Failed to load known spam posts. Reason:")); } function changeUrl(path, search_query, push_state) { var event = new CustomEvent( 'urlchange', { detail: { path: path, query: search_query, push_state: push_state, } }); window.dispatchEvent(event); } function handlePageRoot(_, query_params) { console.log("handlePageRoot", JSON.stringify(query_params)); return loadKnownSpamPosts(); } function showSpamPost(path_data, _) { console.log(path_data); var post_id = path_data[1]; pageInfoText.innerText = 'Loading known spam post ' + post_id; var url = GlobalVars.settings.postgrest_url + '/known_spam_post?text_post_id=eq.' + post_id + '&select=*,known_spam_post_attachment(*)'; var headers = { 'Authorization': 'Bearer ' + GlobalVars.settings.jwt }; var postSectionElem = document.createElement('section'); var mainElem = document.querySelector('main'); mainElem.innerHTML = ''; mainElem.appendChild(renderNav()); mainElem.appendChild(postSectionElem); var h2 = document.createElement('h2'); h2.innerText = 'Known Spam Attachments'; mainElem.appendChild(postSectionElem); mainElem.appendChild(h2); var knownSpamAttachmentsSectionElem = document.createElement('section'); mainElem.append(knownSpamAttachmentsSectionElem); var reasonsSectionElem = document.createElement('section'); mainElem.appendChild(reasonsSectionElem); get(url, cancel, headers) .then(function(response) { var json = JSON.parse(response); console.log(json); pageInfoText.innerText = 'Known Spam Post'; json.forEach(function(p) { var postElem = renderPostElem(p); postSectionElem.appendChild(postElem); reasonsSectionElem.appendChild(renderReasons(p)) }); }) .catch(caught.bind(this, "Failed to load known spam post. Reason:")); url = GlobalVars.settings.postgrest_url + '/known_spam_attachments?post_id=eq.' + post_id; get(url, cancel, headers) .then(function(response) { var json = JSON.parse(response); console.log("known spam attachments:", json); knownSpamAttachmentsSectionElem .appendChild(renderKnownSpamAttachments(json)); }) .catch(caught.bind(this, "Failed to load known spam attachments. Reason:")); } function onUrlChange(e) { var path = e.detail.path.split('/').filter(function(s){ return s.length; }); console.log(e, e.detail, path); if (e.detail.push_state) { var params = Object.assign({ __path: e.detail.path }, e.detail.query); var search_query = new URLSearchParams(params); window.history.pushState(null, "", window.location.origin + window.location.pathname + '?' + search_query.toString()); } resolvePath(ROUTES, path, e.detail.query); } function aClickNav(e) { e.preventDefault(); e.stopPropagation(); console.log(this, this.href); window.history.pushState(null, "", this.href); onRealUrlChange(); } function bindEventHandlers() { window.addEventListener('urlchange', onUrlChange); window.addEventListener('popstate', onRealUrlChange); } function gatherElements() { global_elements = {}; var qs; for (var i=0; i < GlobalElemSelectors.length; i++) { var sel = GlobalElemSelectors[i]; var elem = document.querySelector(sel); global_elements[sel] = elem; } } function parseQueryParams(querystring) { return querystring.split('&') .map(function(part) { return part.split('=').map(decodeURIComponent); }) .reduce(function(acc, pair) { return { [pair[0]]: pair[1] } }, {}); } function onRealUrlChange() { //var path = window.location.pathname; var path; var query_params = parseQueryParams(window.location.search.substring(1)); if (query_params.__path != null) { path = query_params.__path; console.log("NEW PATH", path); } else { path = '/'; } changeUrl(path, query_params); } function initPageInfoText() { pageInfoText = document.createElement('div'); pageInfoText.className = 'info'; prepend(document.querySelector('.info-container'), pageInfoText); } function getSettings() { cancelGlobalEvts(); pageInfoText.innerText = 'Loading settings'; return get('/static/settings.json', cancel) .then(function(response) { var result = JSON.parse(response); console.log("settings response:", result); GlobalVars.settings = result; pageInfoText.innerText = 'Have settings.'; }) .catch(caught.bind(this, "Failed to load JWT: ")); } /** * Init */ initPageInfoText(); // gatherElements(); bindEventHandlers(); // searchInputHandler(); getSettings() .then(onRealUrlChange);