crawl.js

const { get } = require('http')
const { JSDOM } = require('jsdom')

async function crawlPage(baseURL, currentURL, pages) {
  const baseURLObj = new URL(baseURL)
  const currentURLObj = new URL(currentURL)
  if (baseURLObj.hostname !== currentURLObj.hostname) {
    return pages
  }

  const normalizeCurrentURL = normalizeURL(currentURL)
  if (pages[normalizeCurrentURL] > 0) {
    pages[normalizeCurrentURL]++
    return pages
  }

  pages[normalizeCurrentURL] = 1
  console.log(`Actively crawling: ${currentURL}`)

  try {
    const resp = await fetch(currentURL)

    if (resp.status > 399) {
      console.log(`error in fetch with status code: ${resp.status} on page: ${currentURL}`)
      return pages
    }
    const constentType = resp.headers.get('content-type')
    if (!constentType.includes('text/html')) {
      console.log(`Non HTML response, content type: ${constentType} on page: ${currentURL}`)
      return pages
    }

    const htmlBody = await resp.text()
    const nextURLs = getURLsFromHTML(htmlBody, baseURL)

    for (const nextURL of nextURLs) {
      pages = await crawlPage(baseURL, nextURL, pages)
    }
  } catch (err) {
    console.log(`error in fetch: ${err.message}, on page: ${currentURL}`)
  }
  return pages
}

function getURLsFromHTML(htmlBody, baseURL){
  const urls = []
  const dom = new JSDOM(htmlBody)
  const linkElements = dom.window.document.querySelectorAll('a')
  for (const linkElement of linkElements) {
    if (linkElement.href.slice(0, 1) === '/') {
      // relative
      try {
        const urlObj = new URL(`${baseURL}${linkElement.href}`)
        urls.push(urlObj.href)
      } catch (err) {
        console.log(`error with relative url: ${err.message}`)
      }
    } else {
      // absolute
      try {
        const urlObj = new URL(linkElement.href)
        urls.push(urlObj.href)
      } catch (err) {
        console.log(`error with absolute url: ${err.message}`)
      }
    }
  }
  return urls
}

function normalizeURL(urlString) {
  const urlObj = new URL(urlString)
  const hostPath = `${urlObj.hostname}${urlObj.pathname}`
  if (hostPath.length > 0 && hostPath.slice(-1) === '/') {
    return hostPath.slice(0, -1)
  }
  return hostPath
}

module.exports = {
  normalizeURL,
  getURLsFromHTML,
  crawlPage
}