Encode html entities for attribute updates (#406)

website-scraper · May 3, 2020 · c565e14 · c565e14
1 parent 9c9985b
commit c565e14
Show file tree

Hide file tree

Showing 6 changed files with 43 additions and 4 deletions.
diff --git a/lib/resource-handler/html/html-source-element.js b/lib/resource-handler/html/html-source-element.js
@@ -38,7 +38,9 @@ class HtmlSourceElement {
 	 * @param {string} newData
 	 */
 	setData (newData) {
-		this.rule.attr ? this.el.attr(this.rule.attr, newData) : this.el.text(newData);
+		// todo: encode can be removed after https://github.com/cheeriojs/cheerio/issues/957 fixed
+		const escapedData = utils.encodeHtmlEntities(newData);
+		this.rule.attr ? this.el.attr(this.rule.attr, escapedData) : this.el.text(newData);
 	}
 
 	removeIntegrityCheck () {

diff --git a/lib/utils/index.js b/lib/utils/index.js
@@ -146,6 +146,10 @@ function decodeHtmlEntities (text) {
 	return typeof text === 'string' ? htmlEntities.decode(text) : '';
 }
 
+function encodeHtmlEntities (text) {
+	return typeof text === 'string' ? htmlEntities.escape(text) : '';
+}
+
 function clone (obj) {
 	return Object.assign({}, obj);
 }
@@ -188,6 +192,7 @@ module.exports = {
 	getTypeByMime,
 	getTypeByFilename,
 	decodeHtmlEntities,
+	encodeHtmlEntities,
 	clone,
 	extend,
 	union,

diff --git a/...ities-in-url/html-entities-in-url.test.js → ...ional/html-entities/html-entities.test.js b/...ities-in-url/html-entities-in-url.test.js → ...ional/html-entities/html-entities.test.js
@@ -6,7 +6,7 @@ var scrape = require('../../../index');
 var testDirname = __dirname + '/.tmp';
 var mockDirname = __dirname + '/mocks';
 
-describe('Functional: html entities in url', function() {
+describe('Functional: html entities', function() {
 
 	beforeEach(function() {
 		nock.cleanAll();
@@ -27,7 +27,9 @@ describe('Functional: html entities in url', function() {
 		// /fonts?family=Myriad&amp;v=2 => /fonts?family=Myriad&v=2
 		nock('http://example.com/').get('/fonts?family=Myriad&v=2').reply(200, 'fonts.css', {'content-type': 'text/css'});
 		// /?a=1&amp;style-attr.png => /?a=1&style-attr.png
-		nock('http://example.com/').get('/style-attr.png?a=1&style-attr.png').reply(200, 'style-attr.png', {'content-type': 'text/css'});
+		nock('http://example.com/').get('/style-attr.png?a=1&style-attr.png').reply(200, 'style-attr.png');
+		// &quot;style-attr2.png&quot; => style-attr2.png
+		nock('http://example.com/').get('/style-attr2.png').reply(200, 'style-attr2.png');
 		// /?a=1&amp;b=2 => /?a=1&b=2
 		nock('http://example.com/').get('/img.png?a=1&b=2').reply(200, 'img.png');
 		// /test?b=2&amp;c=3&amp;d=4 => /test?b=2&c=3&d=4
@@ -56,10 +58,16 @@ describe('Functional: html entities in url', function() {
 			fs.existsSync(testDirname + '/local/fonts.css').should.be.eql(true);
 			should(fs.readFileSync(testDirname + '/local/fonts.css').toString()).be.eql('fonts.css');
 
-			should(indexHtml).containEql('background: url(\'local/style-attr.png\')');
+			// single quote (') replaced with &#x27; in attribute
+			should(indexHtml).containEql('background: url(&#x27;local/style-attr.png&#x27;)');
 			fs.existsSync(testDirname + '/local/style-attr.png').should.be.eql(true);
 			should(fs.readFileSync(testDirname + '/local/style-attr.png').toString()).be.eql('style-attr.png');
 
+			// double quote (") replaced with &quot; in attribute
+			should(indexHtml).containEql('background: url(&quot;local/style-attr2.png&quot;)');
+			fs.existsSync(testDirname + '/local/style-attr2.png').should.be.eql(true);
+			should(fs.readFileSync(testDirname + '/local/style-attr2.png').toString()).be.eql('style-attr2.png');
+
 			should(indexHtml).containEql('img src="local/img.png');
 			fs.existsSync(testDirname + '/local/img.png').should.be.eql(true);
 			should(fs.readFileSync(testDirname + '/local/img.png').toString()).be.eql('img.png');

diff --git a/...nal/html-entities-in-url/mocks/index.html → ...functional/html-entities/mocks/index.html b/...nal/html-entities-in-url/mocks/index.html → ...functional/html-entities/mocks/index.html
@@ -7,6 +7,7 @@
 </head>
 <body>
     <div style="background: url('http://example.com/style-attr.png?a=1&amp;style-attr.png')"></div>
+    <div style="background: url(&quot;http://example.com/style-attr2.png&quot;)"></div>
     <img src="http://example.com/img.png?a=1&amp;b=2" />
     <a href="?b=2&amp;c=3&amp;d=4">test</a>
 </body>

diff --git a/...onal/html-entities-in-url/mocks/style.css → .../functional/html-entities/mocks/style.css b/...onal/html-entities-in-url/mocks/style.css → .../functional/html-entities/mocks/style.css
diff --git a/test/unit/resource-handler/html.test.js b/test/unit/resource-handler/html.test.js
@@ -267,4 +267,27 @@ describe('ResourceHandler: Html', () => {
 			resource.getText().should.not.containEql('integrity="sha256-X+Q/xqnlEgxCczSjjpp2AUGGgqM5gcBzhRQ0p+EAUEk="');
 		});
 	});
+
+	it('should use html entities for updated attributes', () => {
+		const sources = [
+			{ selector: '[style]', attr: 'style' },
+		];
+		downloadChildrenPaths.onFirstCall().resolves('width: 300px; height: 300px; background-image:url("./images/cat.jpg")');
+		htmlHandler = new HtmlHandler({sources}, {downloadChildrenPaths});
+
+		const html = `
+			<html>
+			<body>
+				 <div style="width: 300px; height: 300px; background-image:url(&quot;http://example.com/cat.jpg&quot;)"></div>
+			</body>
+			</html>
+		`;
+
+		const resource = new Resource('http://example.com', 'index.html');
+		resource.setText(html);
+
+		return htmlHandler.handle(resource).then(() => {
+			resource.getText().should.containEql('style="width: 300px; height: 300px; background-image:url(&quot;./images/cat.jpg&quot;)"');
+		});
+	});
 });