mr-drews-assignment-creator/app/api/fetch-url/route.js

import { NextResponse } from "next/server";
import { htmlToText, extractTitle } from "@/lib/html-to-text";

export const dynamic = "force-dynamic";

export async function POST(request) {
  try {
    const { url } = await request.json();
    let target;
    try {
      target = new URL(String(url || "").trim());
    } catch {
      throw new Error("That doesn't look like a valid URL. Include the full address, e.g. https://example.com/article");
    }
    if (!/^https?:$/.test(target.protocol)) throw new Error("Only http and https URLs are supported.");

    const res = await fetch(target.toString(), {
      headers: {
        "User-Agent": "Mozilla/5.0 (compatible; MrDrewsAssignmentCreator/1.0)",
        "Accept": "text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.8",
      },
      redirect: "follow",
      signal: AbortSignal.timeout(20000),
    });
    if (!res.ok) throw new Error(`The page returned an error (HTTP ${res.status}). It may be behind a login or blocking automated access.`);

    const contentType = res.headers.get("content-type") || "";
    const raw = await res.text();
    let text;
    if (contentType.includes("text/plain")) {
      text = raw;
    } else {
      text = htmlToText(raw);
    }
    text = text.slice(0, 200000);
    if (text.trim().length < 200) {
      throw new Error("Very little readable text was found on that page. It may be mostly images or load its content with JavaScript. Try copying the text and pasting it instead.");
    }
    return NextResponse.json({ title: extractTitle(raw), text, chars: text.length });
  } catch (e) {
    const msg = e?.name === "TimeoutError" ? "Timed out fetching that page (20s)." : String(e.message || e);
    return NextResponse.json({ error: msg }, { status: 400 });
  }
}