From 9b0ddc2ac4205ba00d4465b2a1cc5c5b36c85c8e Mon Sep 17 00:00:00 2001 From: Stephcraft Date: Sat, 15 Jan 2022 14:26:19 -0500 Subject: [PATCH] Added Web scrapper utility v2 --- README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/README.md b/README.md index bec6e0b..ae6ea6f 100644 --- a/README.md +++ b/README.md @@ -62,3 +62,29 @@ var result = await WebScrapping.scrape('https://awesome.url/page', 'cool_scrappe ``` + +## Web scrapper utility v2 +Instead of relying on iframes which tend to cause many COORS errors and slowlyness. V2 utilises `fetch` instead. + +###### Content.js +```js +async function request(method, url, other) { + return await BackgroundTasks.execute('request', { method, url, other }) +} + +var html = await request('GET', 'https://awesome.url/page', { cache: 'force-cache' }) +var doc = HTML.toElement(`${html}`) +// either simulate into an iframe or process the initial source code data... +``` + +###### Background.js +```js +function request(method, url, other) { + // request logic with fetch... +} + +BackgroundTasks.addHandeler('request', async (params) => { + var { method, url, body, other } = params + return await request(method, url, other) +}) +```