betterhuman/visual-test-runner.js

#!/usr/bin/env node
const puppeteer = require("puppeteer");
const fs = require("fs");
const path = require("path");

const DEFAULT_VIEWPORTS = {
  mobile:  { width: 375,  height: 812  },
  tablet:  { width: 768,  height: 1024 },
  desktop: { width: 1440, height: 900  },
};

async function run() {
  const manifestPath = path.resolve(process.cwd(), "test-manifest.json");
  if (!fs.existsSync(manifestPath)) {
    console.error("ERROR: test-manifest.json not found in", process.cwd());
    process.exit(1);
  }

  const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf-8"));
  const baseUrl = manifest.baseUrl || "http://localhost:3000";
  const viewports = manifest.viewports
    ? Object.fromEntries(
        Object.entries(manifest.viewports).map(function (entry) {
          return [entry[0], entry[1]];
        })
      )
    : DEFAULT_VIEWPORTS;

  const resultsDir = path.resolve(process.cwd(), "test-results");
  if (!fs.existsSync(resultsDir)) {
    fs.mkdirSync(resultsDir, { recursive: true });
  }

  const executablePath = process.env.PUPPETEER_EXECUTABLE_PATH || "chromium";
  var browser;
  try {
    browser = await puppeteer.launch({
      headless: "new",
      executablePath: executablePath,
      args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-gpu"],
    });
  } catch (err) {
    console.error("ERROR: Failed to launch browser:", err.message);
    process.exit(1);
  }

  var results = { pages: [], passed: 0, failed: 0, errors: [] };

  for (var pi = 0; pi < (manifest.pages || []).length; pi++) {
    var pageDef = manifest.pages[pi];
    var pageResult = {
      id: pageDef.id,
      url: baseUrl + (pageDef.path || "/"),
      viewports: {},
      passed: true,
    };

    var vpEntries = Object.entries(viewports);
    for (var vi = 0; vi < vpEntries.length; vi++) {
      var vpName = vpEntries[vi][0];
      var vpSize = vpEntries[vi][1];
      var vpResult = {
        viewport: vpName,
        screenshots: [],
        actions: [],
        consoleErrors: [],
        passed: true,
      };

      var browserPage = null;
      try {
        browserPage = await browser.newPage();
        await browserPage.setViewport(vpSize);

        var consoleErrors = [];
        browserPage.on("console", function (msg) {
          if (msg.type() === "error") {
            consoleErrors.push(msg.text());
          }
        });
        browserPage.on("pageerror", function (err) {
          consoleErrors.push(err.message);
        });

        await browserPage.goto(pageResult.url, {
          waitUntil: "networkidle2",
          timeout: 30000,
        });

        if (pageDef.waitFor) {
          await browserPage.waitForSelector(pageDef.waitFor, { timeout: 10000 });
        }

        var initialScreenshot = path.join(
          resultsDir,
          pageDef.id + "-" + vpName + "-initial.png"
        );
        await browserPage.screenshot({
          path: initialScreenshot,
          fullPage: true,
        });
        vpResult.screenshots.push(initialScreenshot);

        var actions = pageDef.actions || [];
        for (var ai = 0; ai < actions.length; ai++) {
          var action = actions[ai];
          var actionResult = { id: action.id, type: action.type, passed: true, error: null };
          try {
            await executeAction(browserPage, action);

            var actionScreenshot = path.join(
              resultsDir,
              pageDef.id + "-" + vpName + "-" + action.id + ".png"
            );
            await browserPage.screenshot({
              path: actionScreenshot,
              fullPage: true,
            });
            vpResult.screenshots.push(actionScreenshot);

            if (action.expectAfter) {
              var valid = await validateExpectations(
                browserPage,
                action.expectAfter
              );
              if (!valid) {
                actionResult.passed = false;
                actionResult.error = "Expectation failed for " + action.id;
              }
            }
          } catch (err) {
            actionResult.passed = false;
            actionResult.error = err.message;
          }

          if (!actionResult.passed) {
            vpResult.passed = false;
          }
          vpResult.actions.push(actionResult);
        }

        vpResult.consoleErrors = consoleErrors;
        if (consoleErrors.length > 0) {
          vpResult.passed = false;
        }
      } catch (err) {
        vpResult.passed = false;
        vpResult.error = err.message;
        results.errors.push({
          page: pageDef.id,
          viewport: vpName,
          error: err.message,
        });
      } finally {
        if (browserPage) {
          await browserPage.close().catch(function () {});
        }
      }

      if (!vpResult.passed) {
        pageResult.passed = false;
      }
      pageResult.viewports[vpName] = vpResult;
    }

    if (pageResult.passed) {
      results.passed++;
    } else {
      results.failed++;
    }
    results.pages.push(pageResult);
  }

  await browser.close();

  var summaryPath = path.join(resultsDir, "summary.json");
  fs.writeFileSync(summaryPath, JSON.stringify(results, null, 2));

  console.log("");
  console.log("=== Visual Test Results ===");
  console.log("Pages tested: " + results.pages.length);
  console.log("Passed: " + results.passed);
  console.log("Failed: " + results.failed);

  for (var ri = 0; ri < results.pages.length; ri++) {
    var p = results.pages[ri];
    var icon = p.passed ? "PASS" : "FAIL";
    console.log("  [" + icon + "] " + p.id + " (" + p.url + ")");
    var vpKeys = Object.keys(p.viewports);
    for (var vk = 0; vk < vpKeys.length; vk++) {
      var vr = p.viewports[vpKeys[vk]];
      if (!vr.passed) {
        console.log("         " + vpKeys[vk] + ": FAIL" + (vr.error ? " - " + vr.error : ""));
        for (var ak = 0; ak < (vr.actions || []).length; ak++) {
          var a = vr.actions[ak];
          if (!a.passed) {
            console.log("           action " + a.id + ": " + a.error);
          }
        }
        if (vr.consoleErrors && vr.consoleErrors.length > 0) {
          console.log("           console errors: " + vr.consoleErrors.length);
        }
      }
    }
  }

  console.log("");
  console.log("Summary written to: " + summaryPath);
  process.exit(results.failed > 0 ? 1 : 0);
}

async function executeAction(page, action) {
  switch (action.type) {
    case "click":
      await page.waitForSelector(action.selector, { timeout: 5000 });
      await page.click(action.selector);
      break;
    case "fill-form":
      var fields = action.fields || [];
      for (var fi = 0; fi < fields.length; fi++) {
        var field = fields[fi];
        await page.waitForSelector(field.selector, { timeout: 5000 });
        await page.click(field.selector, { clickCount: 3 });
        await page.type(field.selector, field.value);
      }
      break;
    case "select":
      await page.waitForSelector(action.selector, { timeout: 5000 });
      await page.select(action.selector, action.value);
      break;
    case "scroll":
      if (action.selector) {
        await page.waitForSelector(action.selector, { timeout: 5000 });
        await page.$eval(action.selector, function (el) {
          el.scrollIntoView({ behavior: "smooth", block: "center" });
        });
      } else {
        await page.evaluate(
          function (x, y) { window.scrollTo(x, y); },
          action.x || 0,
          action.y || 0
        );
      }
      await new Promise(function (r) { setTimeout(r, 500); });
      break;
    case "hover":
      await page.waitForSelector(action.selector, { timeout: 5000 });
      await page.hover(action.selector);
      await new Promise(function (r) { setTimeout(r, 300); });
      break;
    case "wait":
      if (action.selector) {
        await page.waitForSelector(action.selector, {
          timeout: action.timeout || 10000,
        });
      } else {
        await new Promise(function (r) { setTimeout(r, action.duration || 1000); });
      }
      break;
    default:
      throw new Error("Unknown action type: " + action.type);
  }
}

async function validateExpectations(page, expectations) {
  var items = Array.isArray(expectations) ? expectations : [expectations];
  for (var ei = 0; ei < items.length; ei++) {
    var expect = items[ei];
    if (expect.url) {
      var currentUrl = page.url();
      var pattern = new RegExp(expect.url);
      if (!pattern.test(currentUrl)) {
        return false;
      }
    }
    if (expect.visible) {
      var el = await page.$(expect.visible);
      if (!el) return false;
      var isVisible = await page.evaluate(function (s) {
        var e = document.querySelector(s);
        if (!e) return false;
        var r = e.getBoundingClientRect();
        return r.width > 0 && r.height > 0;
      }, expect.visible);
      if (!isVisible) return false;
    }
    if (expect.hidden) {
      var elH = await page.$(expect.hidden);
      if (elH) {
        var isVis = await page.evaluate(function (s) {
          var e = document.querySelector(s);
          if (!e) return false;
          var r = e.getBoundingClientRect();
          return r.width > 0 && r.height > 0;
        }, expect.hidden);
        if (isVis) return false;
      }
    }
    if (expect.text) {
      var textContent = await page.$eval(
        expect.text.selector,
        function (el) { return el.textContent; }
      );
      if (!textContent || !textContent.includes(expect.text.contains)) {
        return false;
      }
    }
  }
  return true;
}

run().catch(function (err) {
  console.error("Fatal error:", err);
  process.exit(1);
});