{
  "slug": "scrape-websites",
  "label": "scrape websites",
  "blurb": "Crawl sites and turn pages into clean text or structured data an agent can use.",
  "keywords": [
    "scrape",
    "scraping",
    "crawl",
    "web data"
  ],
  "categories": [
    "browser-computer",
    "data-analytics",
    "research"
  ],
  "access": [
    "api",
    "mcp"
  ],
  "count": 20,
  "official": 4,
  "url": "https://agentswitchboard.dev/tasks/scrape-websites",
  "browseUrl": "https://agentswitchboard.dev/browse?task=scrape-websites",
  "agentsUrl": "https://agentswitchboard.dev/api/agents?task=scrape-websites",
  "agents": [
    {
      "name": "context.dev",
      "slug": "context-dev",
      "url": "https://agentswitchboard.dev/agents/context-dev",
      "iconUrl": "https://agentswitchboard.dev/icons/context-dev.png?v=9cc9151e",
      "description": "Scrapes, crawls and searches the web for agents through a REST API, SDKs, CLI and hosted MCP server, returning LLM-ready Markdown, structured fields and company brand data.",
      "provider": "context.dev",
      "categories": [
        "browser-computer",
        "research"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Scrape to Markdown: Turns any URL into Markdown, HTML, screenshots, images or CSS-parsed fields in one POST /v1/web/scrape request."
    },
    {
      "name": "Oxylabs MCP",
      "slug": "oxylabs-mcp",
      "url": "https://agentswitchboard.dev/agents/oxylabs-mcp",
      "iconUrl": "https://agentswitchboard.dev/icons/oxylabs-mcp.png?v=2ca92dd8",
      "description": "Lets AI agents scrape any URL with JavaScript rendering and geo-targeting, pull Google and Amazon results, and crawl sites with AI extraction, via the official Oxylabs MCP server.",
      "provider": "Oxylabs",
      "categories": [
        "research",
        "browser-computer"
      ],
      "accessMethods": [
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "bearer",
      "why": "Universal Scraper: Scrapes any URL with optional JavaScript rendering and geo-targeting, returning Markdown, HTML, or extracted links."
    },
    {
      "name": "Firecrawl",
      "slug": "firecrawl",
      "url": "https://agentswitchboard.dev/agents/firecrawl",
      "iconUrl": "https://agentswitchboard.dev/icons/firecrawl.png?v=8c96d4cb",
      "description": "Turn websites into LLM-ready data — crawl, scrape, and convert web pages to clean markdown for AI consumption.",
      "provider": "Firecrawl",
      "categories": [
        "research",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "ScrapeGraphAI",
      "slug": "scrapegraphai",
      "url": "https://agentswitchboard.dev/agents/scrapegraphai",
      "iconUrl": "https://agentswitchboard.dev/icons/scrapegraphai.png?v=e0c23238",
      "description": "Scrapes websites from a natural-language description of the data you want, building the extraction graph automatically and serving results via a hosted API and MCP server.",
      "provider": "ScrapeGraphAI",
      "categories": [
        "research",
        "browser-computer",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Search Scraper: Runs a web search and scrapes the top results into one consolidated structured answer."
    },
    {
      "name": "Crawl4AI",
      "slug": "crawl4ai",
      "url": "https://agentswitchboard.dev/agents/crawl4ai",
      "iconUrl": "https://agentswitchboard.dev/icons/crawl4ai.png?v=b87dccfc",
      "description": "Crawls and scrapes web pages into clean Markdown for RAG pipelines, with CSS, XPath, and LLM-driven structured extraction, deep crawling, and full browser session control.",
      "provider": "UncleCode",
      "categories": [
        "browser-computer",
        "research",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "none",
      "why": "Deep Site Crawling: Traverses whole sites with BFS and best-first strategies under page limits, using prefetch URL discovery and resumable crash…"
    },
    {
      "name": "Nimble",
      "slug": "nimble",
      "url": "https://agentswitchboard.dev/agents/nimble",
      "iconUrl": "https://agentswitchboard.dev/icons/nimble.png?v=14f6c1a5",
      "description": "Searches, extracts, maps and crawls the live web and runs cited research agents, for AI agents that need real-time web data, via REST API, SDKs, a CLI, and a hosted MCP server.",
      "provider": "Nimble",
      "categories": [
        "research",
        "browser-computer"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Site Crawl: Crawls multiple pages of a site with path filtering, subdomain control, and progress tracking."
    },
    {
      "name": "Bright Data MCP",
      "slug": "bright-data-mcp",
      "url": "https://agentswitchboard.dev/agents/bright-data-mcp",
      "iconUrl": "https://agentswitchboard.dev/icons/bright-data-mcp.png?v=6456d332",
      "description": "Enterprise web scraping and browser automation MCP. Lets agents search, crawl, and extract any website without bot detection using Bright Data's proxy network.",
      "provider": "Bright Data",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "mcp"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Page Scraping: Scrape any website including JavaScript-rendered pages through Bright Data's managed proxy network"
    },
    {
      "name": "Apify MCP Server",
      "slug": "apify-mcp",
      "url": "https://agentswitchboard.dev/agents/apify-mcp",
      "iconUrl": "https://agentswitchboard.dev/icons/apify-mcp.png?v=2fc15a7c",
      "description": "Runs thousands of ready-made Apify Actors through an official MCP server — scrape social media, search engines, maps, and e-commerce, then access datasets and run results. MIT, 2k+ GitHub stars.",
      "provider": "Apify",
      "categories": [
        "data-analytics"
      ],
      "accessMethods": [
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Actor Execution: Runs any of thousands of Apify Actors with automatic schema validation to scrape or automate a target site."
    },
    {
      "name": "AgentQL MCP",
      "slug": "agentql-mcp",
      "url": "https://agentswitchboard.dev/agents/agentql-mcp",
      "iconUrl": "https://agentswitchboard.dev/icons/agentql-mcp.png?v=17cde78e",
      "description": "Extracts structured web data using natural language queries — no XPath or selectors needed. Works via Python/JS SDKs, MCP server, and REST API.",
      "provider": "TinyFish",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "browser-extension"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "REST API Extraction: Fetches and parses public-facing web data from any URL via REST API without a browser session running locally."
    },
    {
      "name": "Scrapling",
      "slug": "scrapling",
      "url": "https://agentswitchboard.dev/agents/scrapling",
      "iconUrl": "https://agentswitchboard.dev/icons/scrapling.png?v=0d3e1aef",
      "description": "High-performance Python web scraping library with anti-bot bypass, async support, and smart element auto-matching — faster than Playwright with no browser overhead.",
      "provider": "Scrapling OSS",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "none",
      "why": "Async Scraping: Scrape multiple URLs concurrently with asyncio for high-throughput data collection"
    },
    {
      "name": "Browserless",
      "slug": "browserless",
      "url": "https://agentswitchboard.dev/agents/browserless",
      "iconUrl": "https://agentswitchboard.dev/icons/browserless.png?v=438b2338",
      "description": "Runs hosted or self-hosted headless browsers for agents and scripts, with REST APIs for screenshots, PDFs, and scraping, Puppeteer/Playwright over WebSocket, and an MCP server.",
      "provider": "Browserless",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "api",
        "mcp",
        "cli"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "MCP Browser Agent: Gives AI assistants a stateful browser agent plus smart scraping, page export, crawling, and Lighthouse audits through the…"
    },
    {
      "name": "superagnt_",
      "slug": "superagnt",
      "url": "https://agentswitchboard.dev/agents/superagnt",
      "iconUrl": "https://agentswitchboard.dev/icons/superagnt.png?v=b090bf28",
      "description": "Connects agents to social, web, SEO and lead-enrichment data across 10 platforms and 262 REST endpoints, reachable over one streamable-HTTP MCP server or a keyed REST API.",
      "provider": "agntdata, inc.",
      "categories": [
        "data-analytics",
        "sales-marketing",
        "research"
      ],
      "accessMethods": [
        "mcp",
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "oauth2",
      "why": "Web Scraping: Scrapes any page to clean markdown or structured JSON, searches the web with full page content, and maps a site's URLs."
    },
    {
      "name": "Browserbase",
      "slug": "browserbase",
      "url": "https://agentswitchboard.dev/agents/browserbase",
      "iconUrl": "https://agentswitchboard.dev/icons/browserbase.png?v=1bbd5d24",
      "description": "Managed headless browser infrastructure for AI agents. Run Playwright or Puppeteer at scale with built-in proxies, CAPTCHA solving, and session management.",
      "provider": "Browserbase",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Scrape and interact with web pages at scale with parallel session management"
    },
    {
      "name": "Jina Reader",
      "slug": "jina-reader",
      "url": "https://agentswitchboard.dev/agents/jina-reader",
      "iconUrl": "https://agentswitchboard.dev/icons/jina-reader.png?v=6719c226",
      "description": "Web reader and search API for AI pipelines. Converts any URL to clean markdown, runs grounded search, and powers RAG at scale — no browser automation needed.",
      "provider": "Jina AI",
      "categories": [
        "research"
      ],
      "accessMethods": [
        "api",
        "mcp"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": "Batch Crawl: Crawl entire sites or URL lists in parallel and return structured markdown output for pipelines"
    },
    {
      "name": "Keenable",
      "slug": "keenable",
      "url": "https://agentswitchboard.dev/agents/keenable",
      "iconUrl": "https://agentswitchboard.dev/icons/keenable.png?v=c7bb01f1",
      "description": "Serves a hundred-billion-document web index built for agents rather than human search pages, returning dense structured results over an API and MCP endpoint.",
      "provider": "Keenable",
      "categories": [
        "research",
        "infrastructure",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": "Content Retrieval: Fetches full document content directly from the index without a separate crawl step."
    },
    {
      "name": "Serper",
      "slug": "serper",
      "url": "https://agentswitchboard.dev/agents/serper",
      "iconUrl": "https://agentswitchboard.dev/icons/serper.png?v=9c1ac502",
      "description": "Fast Google search API returning structured SERP results, widely used to give agents fresh web results without scraping.",
      "provider": "Serper",
      "categories": [
        "research"
      ],
      "accessMethods": [
        "api"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Browser Use",
      "slug": "browser-use",
      "url": "https://agentswitchboard.dev/agents/browser-use",
      "iconUrl": "https://agentswitchboard.dev/icons/browser-use.png?v=43f8fd23",
      "description": "Open-source Python library for AI browser automation. Control any website with natural language; cloud adds stealth browsers, proxies, and CAPTCHA solving.",
      "provider": "Browser Use",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": true,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "agent-browser",
      "slug": "agent-browser",
      "url": "https://agentswitchboard.dev/agents/agent-browser",
      "iconUrl": "https://agentswitchboard.dev/icons/agent-browser.png?v=7f93989f",
      "description": "Lets coding agents drive Chrome from a native Rust CLI or stdio MCP server, using accessibility snapshots with element refs, semantic locators, and saved login state.",
      "provider": "Vercel Labs",
      "categories": [
        "browser-computer"
      ],
      "accessMethods": [
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "none",
      "why": null
    },
    {
      "name": "Parallel",
      "slug": "parallel-web",
      "url": "https://agentswitchboard.dev/agents/parallel-web",
      "iconUrl": "https://agentswitchboard.dev/icons/parallel-web.png?v=eb93951f",
      "description": "Gives AI agents web search, URL extraction, and deep research APIs that return cited, structured results, with Python and TypeScript SDKs, a CLI, and hosted MCP servers.",
      "provider": "Parallel Web Systems",
      "categories": [
        "research"
      ],
      "accessMethods": [
        "api",
        "cli",
        "mcp"
      ],
      "verified": true,
      "official": true,
      "featured": false,
      "authType": "apiKey",
      "why": null
    },
    {
      "name": "Tako",
      "slug": "tako",
      "url": "https://agentswitchboard.dev/agents/tako",
      "iconUrl": "https://agentswitchboard.dev/icons/tako.png?v=f26086dc",
      "description": "Live web search plus licensed datasets for agents, covering company financials, macroeconomic indicators, web traffic, and government spending.",
      "provider": "TakoData",
      "categories": [
        "research",
        "data-analytics"
      ],
      "accessMethods": [
        "api",
        "mcp"
      ],
      "verified": true,
      "official": false,
      "featured": false,
      "authType": "apiKey",
      "why": null
    }
  ],
  "relatedTasks": [
    "search-the-web",
    "automate-a-browser",
    "read-pdfs-and-documents",
    "search-my-documents"
  ],
  "stacks": [
    "research-analyst"
  ]
}