Web Scraping with Node.js

mkdir node-scraper && cd node-scraper
npm init -y

# Static scraping
npm install axios cheerio

# Dynamic scraping (downloads Chromium automatically)
npm install puppeteer

# Utilities
npm install fs-extra csv-writer p-limit

node-scraper/
├── scrapers/
│   ├── cheerio-scraper.js   # Static HTML scraping
│   └── puppeteer-scraper.js # Dynamic JS-rendered scraping
├── utils/
│   ├── http.js              # Axios instance with retry logic
│   └── export.js            # CSV / JSON export helpers
├── data/                    # Output directory
└── index.js                 # Entry point

const axios = require('axios');
const cheerio = require('cheerio');

async function scrapeArticles(url) {
  const { data: html } = await axios.get(url, {
    headers: {
      'User-Agent':
        'Mozilla/5.0 (compatible; MyCrawler/1.0; +https://example.com)',
    },
    timeout: 10_000,
  });

  const $ = cheerio.load(html);
  const articles = [];

  // Iterate each article card in the listing
  $('article.post-card').each((_, el) => {
    articles.push({
      title: $(el).find('h2.post-title').text().trim(),
      url: $(el).find('a.post-link').attr('href'),
      date: $(el).find('time').attr('datetime'),
      summary: $(el).find('p.excerpt').text().trim(),
      tags: $(el)
        .find('.tag')
        .map((_, t) => $(t).text().trim())
        .get(),
    });
  });

  return articles;
}

module.exports = { scrapeArticles };

async function scrapeAllPages(baseUrl, maxPages = 10) {
  const allItems = [];
  let page = 1;

  while (page <= maxPages) {
    const url = `${baseUrl}?page=${page}`;
    console.log(`Scraping page ${page}: ${url}`);

    const { data: html } = await axios.get(url, { timeout: 10_000 });
    const $ = cheerio.load(html);

    // Collect items on this page
    $('li.result-item').each((_, el) => {
      allItems.push({
        name: $(el).find('.name').text().trim(),
        price: parseFloat($(el).find('.price').text().replace(/[^0-9.]/g, '')),
        sku: $(el).data('sku'),
      });
    });

    // Stop if there is no "next" link
    const hasNext = $('a[rel="next"]').length > 0;
    if (!hasNext) break;

    page++;
    // Polite delay so we don't hammer the server
    await new Promise(r => setTimeout(r, 1500));
  }

  return allItems;
}

const axios = require('axios');

const client = axios.create({
  timeout: 15_000,
  headers: {
    'User-Agent':
      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept-Language': 'en-US,en;q=0.9',
    Accept: 'text/html,application/xhtml+xml',
  },
});

// Automatic retry on 429 / 5xx with exponential back-off
client.interceptors.response.use(null, async error => {
  const config = error.config;
  config._retryCount = config._retryCount ?? 0;

  const retryableStatus = [429, 500, 502, 503, 504];
  const shouldRetry =
    config._retryCount < 3 &&
    (!error.response || retryableStatus.includes(error.response.status));

  if (!shouldRetry) return Promise.reject(error);

  config._retryCount += 1;
  const delay = 1000 * 2 ** config._retryCount; // 2s, 4s, 8s
  console.log(`Retry ${config._retryCount} for ${config.url} in ${delay}ms`);
  await new Promise(r => setTimeout(r, delay));
  return client(config);
});

module.exports = client;

const pLimit = require('p-limit');
const client = require('./utils/http');
const cheerio = require('cheerio');

// Max 3 concurrent requests at once
const limit = pLimit(3);

async function scrapeProduct(url) {
  const { data: html } = await client.get(url);
  const $ = cheerio.load(html);
  return {
    url,
    title: $('h1.product-title').text().trim(),
    price: $('[data-price]').data('price'),
    description: $('div.product-description').text().trim().slice(0, 500),
    images: $('img.product-image')
      .map((_, el) => $(el).attr('src'))
      .get(),
  };
}

async function scrapeAll(urls) {
  const tasks = urls.map(url =>
    limit(async () => {
      try {
        return await scrapeProduct(url);
      } catch (err) {
        console.error(`Failed: ${url}: ${err.message}`);
        return null;
      }
    })
  );

  const results = await Promise.all(tasks);
  return results.filter(Boolean); // remove nulls from failures
}

module.exports = { scrapeAll };

const { createObjectCsvWriter } = require('csv-writer');
const fs = require('fs-extra');
const path = require('path');

async function saveJSON(data, filename) {
  await fs.ensureDir('./data');
  const filePath = path.join('./data', filename);
  await fs.writeJSON(filePath, data, { spaces: 2 });
  console.log(`Saved ${data.length} records → ${filePath}`);
}

async function saveCSV(data, filename, headers) {
  await fs.ensureDir('./data');
  const writer = createObjectCsvWriter({
    path: path.join('./data', filename),
    header: headers, // [{ id: 'title', title: 'Title' }, ...]
  });
  await writer.writeRecords(data);
  console.log(`CSV written → ./data/${filename}`);
}

module.exports = { saveJSON, saveCSV };

// npm install puppeteer-extra puppeteer-extra-plugin-stealth
const puppeteer = require('puppeteer-extra');
const StealthPlugin = require('puppeteer-extra-plugin-stealth');

puppeteer.use(StealthPlugin());

const browser = await puppeteer.launch({ headless: 'new' });
const page = await browser.newPage();

// Randomise viewport to avoid static fingerprint
await page.setViewport({
  width: 1280 + Math.floor(Math.random() * 100),
  height: 800 + Math.floor(Math.random() * 100),
});

// Pass the navigator.webdriver check
await page.evaluateOnNewDocument(() => {
  Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
});

await page.goto('https://example.com', { waitUntil: 'networkidle2' });

const client = require('./utils/http');
const cheerio = require('cheerio');
const { saveJSON, saveCSV } = require('./utils/export');
const pLimit = require('p-limit');

const BASE = 'https://jobs.example.com';
const limit = pLimit(2);

async function getListingUrls(page = 1) {
  const { data: html } = await client.get(`${BASE}/jobs?page=${page}`);
  const $ = cheerio.load(html);
  return $('a.job-link')
    .map((_, el) => BASE + $(el).attr('href'))
    .get();
}

async function scrapeJob(url) {
  await new Promise(r => setTimeout(r, 800 + Math.random() * 1200));
  const { data: html } = await client.get(url);
  const $ = cheerio.load(html);
  return {
    title: $('h1.job-title').text().trim(),
    company: $('span.company').text().trim(),
    location: $('span.location').text().trim(),
    salary: $('span.salary').text().trim() || 'Not disclosed',
    posted: $('time').attr('datetime'),
    description: $('div.job-description').text().trim().slice(0, 1000),
    url,
  };
}

async function run() {
  console.log('Starting job scraper...');
  const urls = [];

  for (let p = 1; p <= 5; p++) {
    const pageUrls = await getListingUrls(p);
    urls.push(...pageUrls);
    await new Promise(r => setTimeout(r, 1500));
  }

  console.log(`Found ${urls.length} job listings`);

  const jobs = await Promise.all(
    urls.map(url => limit(() => scrapeJob(url).catch(() => null)))
  );
  const valid = jobs.filter(Boolean);

  await saveJSON(valid, 'jobs.json');
  await saveCSV(valid, 'jobs.csv', [
    { id: 'title', title: 'Title' },
    { id: 'company', title: 'Company' },
    { id: 'location', title: 'Location' },
    { id: 'salary', title: 'Salary' },
    { id: 'posted', title: 'Posted' },
    { id: 'url', title: 'URL' },
  ]);

  console.log(`Done! Scraped ${valid.length} jobs.`);
}

run().catch(console.error);

// Run in headed mode to watch what's happening
const browser = await puppeteer.launch({
  headless: false,        // show the browser window
  slowMo: 250,            // slow down each action by 250ms
  devtools: true,         // open DevTools automatically
});

// Take a screenshot to debug the current page state
await page.screenshot({ path: 'debug.png', fullPage: true });

// Dump the full rendered HTML
const html = await page.content();
require('fs').writeFileSync('debug.html', html);