This repository was archived by the owner on Nov 6, 2023. It is now read-only.
Repository navigation
Add HSTS-based rewrites to the Node rewriter. #4462
Closed
Closed
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,26 @@ | ||
| // A tool to transform transport_security_state_static.json into one big trivial | ||
| // ruleset on stdout. | ||
| // To get transport_security_state_static.json: | ||
| // curl https://chromium.googlesource.com/chromium/src/+/master/net/http/transport_security_state_static.json?format=TEXT \ | ||
| // | base64 -d > transport_security_state_static.json | ||
| var fs = require('fs'); | ||
|
|
||
| if (process.argv.length != 3) { | ||
| console.error('Provide path to transport_security_state_static.json as only argument.'); | ||
| return; | ||
| } | ||
| var contents = fs.readFileSync(process.argv[2], 'utf8'); | ||
| contents = contents.split('\n').filter(line => !/^\s*\/\//.test(line)).join('\n'); | ||
| var secStateStatic = JSON.parse(contents); | ||
| console.log('<ruleset name="hsts">') | ||
| secStateStatic.entries.forEach(function(entry) { | ||
| if (entry.mode != "force-https") { | ||
| return; | ||
| } | ||
| console.log(' <target host="' + entry.name + '" />'); | ||
| if (entry.include_subdomains) { | ||
| console.log(' <target host="*.' + entry.name + '" />'); | ||
| } | ||
| }); | ||
| console.log(' <rule from="^http:" to="https:" />'); | ||
| console.log('</ruleset>'); | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,10 +1,10 @@ | ||
| { | ||
| "name" : "https-everywhere-rewriter", | ||
| "version" : "1.0.0", | ||
| "dependencies" : { | ||
| "xmldom" : "0.1.17", | ||
| "URIjs" : "1.11.2", | ||
| "readdirp" : "0.3.2", | ||
| "event-stream" : "3.0.20" | ||
| "name": "https-everywhere-rewriter", | ||
| "version": "1.1.0", | ||
| "dependencies": { | ||
| "URIjs": "*", | ||
| "event-stream": "^3.3.2", | ||
| "readdirp": "^2.0.0", | ||
| "xmldom": "^0.1.22" | ||
| } | ||
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -24,7 +24,7 @@ var path = require("path"), | |
|
|
||
| URI = require("URIjs"); | ||
|
|
||
| var ruleSets = null; | ||
| var ruleSets = new rules.RuleSets({}); | ||
|
|
||
| /** | ||
| * For a given directory, recursively edit all files in it that match a filename | ||
|
|
@@ -48,7 +48,6 @@ function processDir(dir) { | |
| .on('error', function (err) { console.error('fatal error', err); }) | ||
| .pipe(es.mapSync(function (entry) { | ||
| var filename = path.join(dir, entry.path); | ||
| console.log("Rewriting " + filename); | ||
| try { | ||
| processFile(filename); | ||
| } catch(e) { | ||
|
|
@@ -57,27 +56,16 @@ function processDir(dir) { | |
| })); | ||
| } | ||
|
|
||
| /** | ||
| * Overwrite the default URI find_uri_expression with a modified one that | ||
| * mitigates a catastrophic backtracking issue common in CSS. | ||
| * The workaround was to insist that URLs start with http, since those are the | ||
| * only ones we want to rewrite anyhow. Note that this may still go exponential | ||
| * on certain inputs. http://www.regular-expressions.info/catastrophic.html | ||
| * Example string that blows up URI.withinString: | ||
| * image:url(http://img.youtube.com/vi/x7f | ||
| */ | ||
| URI.find_uri_expression = /\b((?:http:(?:\/{1,3}|[a-z0-9%])|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}\/)(?:[^\s()<>]+)+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:'".,<>?«»“”‘’]))/ig; | ||
|
|
||
| function processFile(filename) { | ||
| var contents = fs.readFileSync(filename, 'utf8'); | ||
| var rewrittenFile = URI.withinString(contents, function(url) { | ||
| console.log("Found ", url); | ||
| var uri = new URI(url); | ||
| if (uri.protocol() != 'http') return url; | ||
|
|
||
| uri.normalize(); | ||
| var rewritten = ruleSets.rewriteURI(uri.toString(), uri.host()); | ||
| if (rewritten) { | ||
| console.log('Rewrote', url, 'in', filename); | ||
| return rewritten; | ||
| } else { | ||
| return url; | ||
|
|
@@ -92,16 +80,15 @@ function processFile(filename) { | |
| /** | ||
| * Load all rulesets for rewriting. | ||
| */ | ||
| function loadRuleSets() { | ||
| console.log("Loading rules..."); | ||
| var fileContents = fs.readFileSync(path.join(__dirname, '../pkg/crx/rules/default.rulesets'), 'utf8'); | ||
| function loadRuleSets(filename) { | ||
| console.log("Loading rules from", filename); | ||
| var fileContents = fs.readFileSync(filename, 'utf8'); | ||
| var xml = new DOMParser().parseFromString(fileContents, 'text/xml'); | ||
| ruleSets = new rules.RuleSets({}); | ||
| ruleSets.addFromXml(xml); | ||
| } | ||
|
|
||
| function usage() { | ||
| console.log("Usage: \n nodejs rewriter.js /path/to/my/webapp \n"); | ||
| console.log("Usage:\n nodejs rewriter.js /path/to/my/webapp\n"); | ||
| process.exit(1); | ||
| } | ||
|
|
||
|
|
@@ -120,7 +107,16 @@ for (var i = 2; i < process.argv.length; i++) { | |
| } | ||
| } | ||
|
|
||
| loadRuleSets(); | ||
| var hstsFilename = path.join(__dirname, 'hsts.xml'); | ||
| try { | ||
| fs.statSync(hstsFilename); | ||
| } catch (e) { | ||
| console.error('hsts.xml not found. Run hsts-transform.js.'); | ||
|
Member
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Any reason not to just output to |
||
| return; | ||
| } | ||
| loadRuleSets(hstsFilename); | ||
|
|
||
| loadRuleSets(path.join(__dirname, '../pkg/crx/rules/default.rulesets')); | ||
| for (var i = 2; i < process.argv.length; i++) { | ||
| processDir(process.argv[i]); | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
You'll probably want to make this
entry.name.trim()because the upstreamtransport_security_state_static.jsonmistakenly contains spaces after some domains.