From 855553acba132562784e9a8e4a5e76bb284e6031 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:41:21 +0000 Subject: [PATCH 1/23] Hide per-script action buttons until functionality works --- diff --git a/html/display_panel/content/main_panel.js b/html/display_panel/content/main_panel.js index 7c1170d..8b20ff5 100644 --- a/html/display_panel/content/main_panel.js +++ b/html/display_panel/content/main_panel.js @@ -78,9 +78,9 @@ function new_debug_button(name_text,callback){ */ function write_elements(data,name,color){ var url = data["url"]; - var button_html = '
'; - var button_html_2 = '
'; - var button_html_3 = '
'; + var button_html = '
'; + var button_html_2 = '
'; + var button_html_3 = '
'; var heading = document.getElementById(name).getElementsByTagName("h2")[0]; var list = document.getElementById(name).getElementsByTagName("ul")[0]; if(typeof(data[name]) == "undefined" || data[name].length == 0){ From 6e31bd9b74f260c050f78b97752bc6a9359c2a69 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:44:21 +0000 Subject: [PATCH 2/23] Evaluate the original script, not the mangled copy for reserved access evaluation --- diff --git a/main_background.js b/main_background.js index 8d6052d..26167d4 100644 --- a/main_background.js +++ b/main_background.js @@ -817,8 +817,8 @@ function evaluate(script,name){ } else{ return [flag,reason]; } - - var final = full_evaluate(temp); + + var final = full_evaluate(script); // final[1] = final[1] + "
"; return final; } From ecd59d3bed9d53542ec31fd3dc0c96b5e7eb1084 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:45:39 +0000 Subject: [PATCH 3/23] Do not run license_read on whitelisted scripts --- diff --git a/main_background.js b/main_background.js index 26167d4..dfdfd18 100644 --- a/main_background.js +++ b/main_background.js @@ -933,6 +933,7 @@ function get_script(response,url,tabid,wl,index=-1){ if(unused_data[tabid] === undefined){ unused_data[tabid] = {"url":url,"accepted":[],"blocked":[]}; } + var edited; var tok_index = url.split("/").length; var scriptname = url.split("/")[tok_index-1]; if(wl == true){ @@ -948,10 +949,11 @@ function get_script(response,url,tabid,wl,index=-1){ } else{ resolve("\n/*\n LibreJS: Script whitelisted by user (From a URL found in comma seperated whitelist)\n*/\n"+response); } + edited = [true,response,"Page is whitelisted in preferences"]; + }else{ + edited = license_read(response,scriptname); } - var src_hash = hash(response); - var edited = license_read(response,scriptname); var verdict = edited[0]; var popup_res; var domain = get_domain(url); From 35fd1d805c16bebbeb20224c9a18da919aef70b0 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:46:50 +0000 Subject: [PATCH 4/23] Use multi-part fetch mechanism for read_script --- diff --git a/main_background.js b/main_background.js index dfdfd18..b0c8d9b 100644 --- a/main_background.js +++ b/main_background.js @@ -1068,9 +1068,10 @@ function read_script(a){ var filter = webex.webRequest.filterResponseData(a.requestId); var decoder = new TextDecoder("utf-8"); var encoder = new TextEncoder(); + var str = ""; - filter.ondata = event => { - var str = decoder.decode(event.data, {stream: true}); + filter.onstop = event => { + console.log("read_script "+a.url); var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ var edit_script; @@ -1086,6 +1087,9 @@ function read_script(a){ }); }); } + filter.ondata = event => { + str += decoder.decode(event.data, {stream: true}); + } return {}; } From 7994332e313c801c5167ad4bd42e654f7d3dfc0f Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:48:34 +0000 Subject: [PATCH 5/23] Always clear unused_data[tabid] before script evaluation --- diff --git a/main_background.js b/main_background.js index b0c8d9b..df931de 100644 --- a/main_background.js +++ b/main_background.js @@ -1255,16 +1255,11 @@ function edit_html(html,url,tabid,wl){ * */ function read_document(a){ - var GA = test_GA(a); if(GA != false){ return GA; } - - if(unused_data[a["tabId"]] !== undefined && unused_data[a["tabId"]]["url"] != get_domain(a["url"])){ - delete unused_data[a["tabId"]]; - dbg_print("Page Changed!!!"); - } + delete unused_data[a["tabId"]]; var str = ""; var filter = webex.webRequest.filterResponseData(a.requestId); var decoder = new TextDecoder("utf-8"); From 6e2b4fb5cf520da4a7a84a9c3a27e1228369563f Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 03 2018 23:51:33 +0000 Subject: [PATCH 6/23] HACK: do not process non-html documents in read_document Processing them breaks reading js documents directly. The method is crude it needs to be generalized for other mime-types, in particular librejs breaks visualizing any binary files (e.g. images) --- diff --git a/main_background.js b/main_background.js index df931de..a2ab782 100644 --- a/main_background.js +++ b/main_background.js @@ -1269,10 +1269,14 @@ function read_document(a){ } filter.onstop = event => { var test = new ArrayBuffer(); - var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ var edit_page; + if(! str.includes(" Date: May 11 2018 17:18:33 +0000 Subject: [PATCH 7/23] Disable whitelisted and blacklisted sections on display panel for now --- diff --git a/html/display_panel/content/display-panel.html b/html/display_panel/content/display-panel.html index 79126a0..4f2da4f 100644 --- a/html/display_panel/content/display-panel.html +++ b/html/display_panel/content/display-panel.html @@ -55,7 +55,7 @@
-
+ @@ -67,7 +67,7 @@

    -
    + From e038916fe397368a7fee8ffa86dbd6c025b8d993 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:21:18 +0000 Subject: [PATCH 8/23] Regexp matches at stripLicenseToRegexp should be non-global Global matching results in alternating true/false results as explained in https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/RegExp/test (Using test() on a regex with the global flag) --- diff --git a/legacy_license_check.js b/legacy_license_check.js index 00d341e..f783129 100644 --- a/legacy_license_check.js +++ b/legacy_license_check.js @@ -40,7 +40,7 @@ var stripLicenseToRegexp = function (license) { item = license.licenseFragments[i]; item.regex = match_utils.removeNonalpha(item.text); item.regex = new RegExp( - match_utils.replaceTokens(item.regex), 'g'); + match_utils.replaceTokens(item.regex), ''); } return license; }; From 12c4c697f00a703b768ab553e486be9d9f457dee Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:36:56 +0000 Subject: [PATCH 9/23] Set dbg_print() on main_panel.js and clean default output --- diff --git a/html/display_panel/content/main_panel.js b/html/display_panel/content/main_panel.js index 8b20ff5..3086859 100644 --- a/html/display_panel/content/main_panel.js +++ b/html/display_panel/content/main_panel.js @@ -31,6 +31,18 @@ * * (Use the variable "webex" for all API calls after calling this) */ + +var PRINT_DEBUG = false; +function dbg_print(a,b){ + if(PRINT_DEBUG == true){ + if(b === undefined){ + console.log(a); + } else{ + console.log(a,b); + } + } +} + var webex; function set_webex(){ if(typeof(browser) == "undefined"){ @@ -183,8 +195,8 @@ function generate_HTML(blocked_data){ for(var i = 0; i < to_clr.length; i++){ to_clr[i].innerHTML = ""; } - console.log("REGEN HTML:"); - console.log(blocked_data); + dbg_print("REGEN HTML:"); + dbg_print(blocked_data); write_elements(blocked_data,"accepted","green"); write_elements(blocked_data,"whitelisted","green"); write_elements(blocked_data,"blocked","red"); From 37559e3cd93f58f23a7a3d3ec413fcbae22656e7 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:37:37 +0000 Subject: [PATCH 10/23] Make legacy_license_check return the matched license name. Also improved logging. --- diff --git a/legacy_license_check.js b/legacy_license_check.js index f783129..5068085 100644 --- a/legacy_license_check.js +++ b/legacy_license_check.js @@ -74,12 +74,12 @@ var search_table = function(stripped_comment){ max_i = data.licenses[license].licenseFragments.length; for (i = 0; i < max_i; i++) { if (frag[i].regex.test(stripped)) { - console.log(data.licenses[license].licenseName); - return true; + //console.log(data.licenses[license].licenseName); + return data.licenses[license].licenseName; } } } - console.log("No license found."); + console.log("No global license found."); return false; } From 60653666f2504ae7979880e4fcdd90f28facdc3c Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:38:24 +0000 Subject: [PATCH 11/23] Generalize comment styles for @license matching --- diff --git a/pattern_utils.js b/pattern_utils.js index b3e84d6..ce20842 100644 --- a/pattern_utils.js +++ b/pattern_utils.js @@ -33,7 +33,7 @@ exports.patternUtils = { return str.replace(regex, ''); }, removeWhitespace: function (str) { - return str.replace(/\s+/gmi, ''); + return str.replace(/\/\//gmi, '').replace(/\*/gmi, '').replace(/\s+/gmi, ''); }, replaceTokens: function (str) { var regex = /<.*?>/gi; From 025c8bbc3b81f7c90681f3a020b9c3b41ae192ca Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:39:52 +0000 Subject: [PATCH 12/23] Generalize gpl-3.0 license text --- diff --git a/license_definitions.js b/license_definitions.js index 26e56a0..1ea9fba 100644 --- a/license_definitions.js +++ b/license_definitions.js @@ -66,7 +66,7 @@ exports.licenses = { 'magnet:?xt=urn:btih:1f739d935676111cfff4b4693e3816e664797050&dn=gpl-3.0.txt' ], licenseFragments: [ - {text: "The JavaScript code in this page is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License (GNU GPL) as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The code is distributed WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU GPL for more details. As additional permission under GNU GPL version 3 section 7, you may distribute non-source (e.g., minimized or compacted) forms of that code without the copy of the GNU GPL normally required by section 4, provided you include this license notice and a URL through which recipients can access the Corresponding Source.", type: type.SHORT}, + {text: " is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License (GNU GPL) as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. The code is distributed WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU GPL for more details. As additional permission under GNU GPL version 3 section 7, you may distribute non-source (e.g., minimized or compacted) forms of that code without the copy of the GNU GPL normally required by section 4, provided you include this license notice and a URL through which recipients can access the Corresponding Source.", type: type.SHORT}, {text: " is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.", type: type.SHORT}] }, From 598ca7b8d266f5d60a306e92993bc79d97d5f9aa Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:45:18 +0000 Subject: [PATCH 13/23] Enable legacy license matching and hash whitelist matching at license_read() --- diff --git a/main_background.js b/main_background.js index a2ab782..050f41e 100644 --- a/main_background.js +++ b/main_background.js @@ -860,6 +860,13 @@ function license_read(script_src, name){ var nontrivial_status; var parts_denied = false; var parts_accepted = false; + var license = legacy_license_lib.check(script_src); + if(license != false){ + return [true,script_src,"Licensed under: "+license]; + } + if (default_whitelist[hash(script_src)]){ + return [true,script_src,"Common script known to be free software."]; + } while(true){ // TODO: refactor me // TODO: support multiline comments var matches = /\/\/\s*?(@license)\s([\S]+)\s([\S]+$)/gm.exec(unedited_src); From d71fe0501fc2a792b65f76145550f040201bd386 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:46:51 +0000 Subject: [PATCH 14/23] Generalize comment styles for @license matching at license_read(), and avoid parsing empty whitespace sections --- diff --git a/main_background.js b/main_background.js index 050f41e..7a856b8 100644 --- a/main_background.js +++ b/main_background.js @@ -869,7 +869,11 @@ function license_read(script_src, name){ } while(true){ // TODO: refactor me // TODO: support multiline comments - var matches = /\/\/\s*?(@license)\s([\S]+)\s([\S]+$)/gm.exec(unedited_src); + var matches = /\/[\/\*]\s*?(@license)\s([\S]+)\s([\S]+$)/gm.exec(unedited_src); + var empty = /[^\s]/gm.exec(unedited_src); + if(empty == null){ + return [true,edited_src,reason_text]; + } if(matches == null){ nontrivial_status = evaluate(unedited_src,name); if(nontrivial_status[0] == true){ From eb372d366b1896dba3274200d9a6992b7e2a18ff Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:50:22 +0000 Subject: [PATCH 15/23] Prevent evaluation of external scripts, as they are always nontrivial --- diff --git a/main_background.js b/main_background.js index 7a856b8..de24a4a 100644 --- a/main_background.js +++ b/main_background.js @@ -875,7 +875,10 @@ function license_read(script_src, name){ return [true,edited_src,reason_text]; } if(matches == null){ - nontrivial_status = evaluate(unedited_src,name); + if (external) + return [false,edited_src,"External script with no known license."]; + else + nontrivial_status = evaluate(unedited_src,name); if(nontrivial_status[0] == true){ parts_accepted = true; edited_src += unedited_src; @@ -903,7 +906,10 @@ function license_read(script_src, name){ // sponge console.log("before:"); console.log(before); - nontrivial_status = evaluate(before,name); + if (external) + nontrivial_status = [true, "External script with no known license"] + else + nontrivial_status = evaluate(before,name); if(nontrivial_status[0] == true){ parts_accepted = true; edited_src += before; From f0fb8061ce5ac4c23ea1a2cbf94c4aa1a3f33897 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 17:56:43 +0000 Subject: [PATCH 16/23] Correct tab and badge initialization to prevent race conditions --- diff --git a/main_background.js b/main_background.js index de24a4a..ada6230 100644 --- a/main_background.js +++ b/main_background.js @@ -993,16 +993,7 @@ function get_script(response,url,tabid,wl,index=-1){ webex.browserAction.setBadgeBackgroundColor({ color: "red", tabId: tabid - }); - } else{ - webex.browserAction.setBadgeText({ - text: "✓", - tabId: tabid }); - webex.browserAction.setBadgeBackgroundColor({ - color: "green", - tabId: tabid - }); } if(verdict == true){ @@ -1276,7 +1267,6 @@ function read_document(a){ if(GA != false){ return GA; } - delete unused_data[a["tabId"]]; var str = ""; var filter = webex.webRequest.filterResponseData(a.requestId); var decoder = new TextDecoder("utf-8"); @@ -1285,6 +1275,15 @@ function read_document(a){ dbg_print("%c Error in getting document","color:red"); } filter.onstop = event => { + delete unused_data[a["tabId"]]; + webex.browserAction.setBadgeText({ + text: "✓", + tabId: a["tabId"] + }); + webex.browserAction.setBadgeBackgroundColor({ + color: "green", + tabId: a["tabId"] + }); var test = new ArrayBuffer(); var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ From 0166d8c292987a523f708d95188e81952074a990 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 18:00:33 +0000 Subject: [PATCH 17/23] Refactor whitelisting at read_script() --- diff --git a/main_background.js b/main_background.js index ada6230..a8a0536 100644 --- a/main_background.js +++ b/main_background.js @@ -1082,13 +1082,7 @@ function read_script(a){ console.log("read_script "+a.url); var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ - var edit_script; - if(whitelisted == true){ - // Doesn't matter if this is accepted or blocked, it will still be whitelisted - edit_script = get_script(str,a.url,a["tabId"],true); - } else{ - edit_script = get_script(str,a.url,a["tabId"],false); - } + var edit_script = get_script(str,a.url,a["tabId"],whitelisted,-1); edit_script.then(function(edited){ filter.write(encoder.encode(edited)); filter.disconnect(); From 20c6892732eb36952ebf9314172eb450e631ab21 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 18:03:47 +0000 Subject: [PATCH 18/23] Check global license and add to popup. Prevent parsing inline scripts if there is a global license. --- diff --git a/main_background.js b/main_background.js index a8a0536..b1c482c 100644 --- a/main_background.js +++ b/main_background.js @@ -1187,8 +1187,12 @@ function edit_html(html,url,tabid,wl){ break; } } - if(read_metadata(meta_element) || legacy_license_lib.check(first_script_src)){ + var license = legacy_license_lib.check(first_script_src); + if(read_metadata(meta_element) || license != false ){ console.log("Valid license for intrinsic events found"); + add_popup_entry(tabid,url,{"url":url,"accepted":[url,"Global license for the page: "+license]}); + // Do not process inline scripts + scripts=""; }else{ // Deal with intrinsic events var has_intrinsic_events = []; From 673ebdf926260cc3b6c49acc187e0fc28bf482fa Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 18:05:51 +0000 Subject: [PATCH 19/23] Do not process non-javascript scripts (json, templates, etc) --- diff --git a/main_background.js b/main_background.js index b1c482c..00aae37 100644 --- a/main_background.js +++ b/main_background.js @@ -1230,24 +1230,22 @@ function edit_html(html,url,tabid,wl){ dbg_print("Analyzing "+total_scripts+" inline scripts..."); for(var i = 0; i < scripts.length; i++){ - if(scripts[i].src == ""){ - var edit_script = get_script(scripts[i].innerHTML,url,tabid,wl,i); - edit_script.then(function(edited){ - var edited_source = edited[0]; - var unedited_source = html_doc.scripts[edited[1]].innerHTML.trim(); - - html_doc.scripts[edited[1]].innerHTML = edited_source; - - amt_scripts++; - - if(amt_scripts >= total_scripts){ - resolve(remove_noscripts(html_doc)); - } - - }); + if (scripts[i].src == ""){ + if (scripts[i].type=="" || scripts[i].type=="text/javascript"){ + var edit_script = get_script(scripts[i].innerHTML,url,tabid,wl,i); + edit_script.then(function(edited){ + var edited_source = edited[0]; + var unedited_source = html_doc.scripts[edited[1]].innerHTML.trim(); + html_doc.scripts[edited[1]].innerHTML = edited_source; + + }); + } + amt_scripts++; + if(amt_scripts >= total_scripts){ + resolve(remove_noscripts(html_doc)); + } } } - if(total_scripts == 0){ dbg_print("Nothing to analyze."); resolve(remove_noscripts(html_doc)); From 7b3839d22a8e6c0fd763eb9f2151d500dc6d03f9 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 18:07:55 +0000 Subject: [PATCH 20/23] Correct hack that prevents parsing non-html documents. Still needs work. --- diff --git a/main_background.js b/main_background.js index 00aae37..a5b43be 100644 --- a/main_background.js +++ b/main_background.js @@ -1284,10 +1284,12 @@ function read_document(a){ var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ var edit_page; - if(! str.includes(" Date: May 11 2018 18:16:17 +0000 Subject: [PATCH 21/23] Made less verbose when debug is off --- diff --git a/main_background.js b/main_background.js index a5b43be..518901d 100644 --- a/main_background.js +++ b/main_background.js @@ -898,14 +898,14 @@ function license_read(script_src, name){ } // sponge - console.log("undedited_src:"); - console.log(unedited_src); - console.log(matches); - console.log("chopping at " + matches["index"] + "."); + dbg_print("undedited_src:"); + dbg_print(unedited_src); + dbg_print(matches); + dbg_print("chopping at " + matches["index"] + "."); var before = unedited_src.substring(0,matches["index"]); // sponge - console.log("before:"); - console.log(before); + dbg_print("before:"); + dbg_print(before); if (external) nontrivial_status = [true, "External script with no known license"] else @@ -1079,7 +1079,7 @@ function read_script(a){ var str = ""; filter.onstop = event => { - console.log("read_script "+a.url); + dbg_print("read_script "+a.url); var res = test_url_whitelisted(a.url); res.then(function(whitelisted){ var edit_script = get_script(str,a.url,a["tabId"],whitelisted,-1); From 107d7ebdd42061cd16fd9a00edd4fd4201ba59a2 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 20:23:03 +0000 Subject: [PATCH 22/23] Add basic time profiling to dbg_print --- diff --git a/main_background.js b/main_background.js index 518901d..16b58d3 100644 --- a/main_background.js +++ b/main_background.js @@ -35,9 +35,11 @@ console.log("main_background.js"); */ var DEBUG = false; // debug the JS evaluation var PRINT_DEBUG = false; // Everything else +var time = Date.now(); function dbg_print(a,b){ if(PRINT_DEBUG == true){ + console.log("Time spent so far: " + (Date.now() - time)/1000 + " seconds"); if(b === undefined){ console.log(a); } else{ @@ -1271,6 +1273,7 @@ function read_document(a){ dbg_print("%c Error in getting document","color:red"); } filter.onstop = event => { + time = Date.now(); delete unused_data[a["tabId"]]; webex.browserAction.setBadgeText({ text: "✓", From 0c09f2fd75973982808bc0732da8f26da6b3a682 Mon Sep 17 00:00:00 2001 From: Ruben Rodriguez Date: May 11 2018 20:36:29 +0000 Subject: [PATCH 23/23] Bumped version number to 7.14 --- diff --git a/manifest.json b/manifest.json index 396b82d..883c7fc 100644 --- a/manifest.json +++ b/manifest.json @@ -2,7 +2,7 @@ "manifest_version": 2, "name": "GNU LibreJS [webExtensions]", "short_name": "LibreJS [experimental]", - "version": "7.12", + "version": "7.14", "author": "various", "description": "Only allows free and/or trivial Javascript to run.", "applications": {