Honestly, I just deployed a docker container for this in my gluetun AI frontend stack and then tied a python script in that lets openwebUI send the call from Qwen to crawl4ai. Here is the crawl4AI section (ports are up with gluetun so I still have access to the webUI for myself, but I have never personally used the UI and just let OpenWebUI handle it

  crawl4ai:
    image: unclecode/crawl4ai:latest
    container_name: crawl4ai
    network_mode: service:gluetun
    environment:
      - CRAWL4AI_API_TOKEN=${CRAWL4AI_API_TOKEN:-} 
      - MAX_CONCURRENT_TASKS=5
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:11235/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 3
    depends_on:
      gluetun:
        condition: service_healthy
    restart: unless-stopped

Here is the python script I put into OpenwebUI tools that handles the scrape and sending to the scout model for summarization (built with some help from Qwen and Gemini to get it working)

"""
title: Scout Scraper Tool
description: Scrapes a website, tokenizes to prevent context overflow, and uses a scout AI to extract data.
requirements: tiktoken, requests, pydantic
version: 1.0.1
"""

import requests
import json
import tiktoken
from pydantic import BaseModel, Field
from typing import Optional

class Tools:
    def __init__(self):
        # Crawl4AI setup
        self.crawl_api_url = "http://localhost:11235/crawl"
        self.crawl_api_token = (
            "<crawl4AI_API_token_Here>"
        )

        # Scout LLM setup (Llama-3.2-3B on Llama.cpp)
        self.scout_api_url = "http://<IPaddress:port_for_llama.cpp_server>/v1/chat/completions"

        # Initialize the tokenizer
        self.tokenizer = tiktoken.get_encoding("cl100k_base")

    def scrape_and_scout(self, url: str, query_context: str) -> str:
        """
        Scrapes a website and uses a scout AI to extract specific technical information.
        Use this when standard search snippets lack sufficient depth.
        :param url: The full URL to scrape.
        :param query_context: Specific instructions on what facts or code to extract from the page.
        """
        headers = {
            "Authorization": f"Bearer {self.crawl_api_token}",
            "Content-Type": "application/json",
        }
        payload = {
            "urls": [url],
            "priority": 10,
            "magic_mode": True,
            "wait_for": "networkidle",
        }

        try:
            # 1. Scrape the URL
            crawl_resp = requests.post(
                self.crawl_api_url, headers=headers, json=payload, timeout=30
            )
            crawl_resp.raise_for_status()
            data = crawl_resp.json()

            if not (data.get("success") and data.get("results")):
                return f"Scrape failed: {data.get('error', 'Unknown error')}"

            result = data["results"][0]

            # --- DATA CLEANING FIX v2 ---
            md_data = result.get("markdown", {})

            if isinstance(md_data, dict):
                # Forums get wiped by magic_mode. Always grab raw_markdown first.
                markdown = md_data.get("raw_markdown", "")
                if not markdown:
                    markdown = md_data.get("fit_markdown", "")
            else:
                markdown = str(md_data)

            # Absolute fallback: if markdown fails, grab the raw text/html
            if not markdown or markdown.strip() in ["", "None"]:
                markdown = result.get("text", result.get("html", ""))

            if not markdown or markdown.strip() in ["", "None"]:
                return "Scrape successful, but the page returned absolutely zero text."
            # -------------------------

            # 2. Tokenizer Fallback / Truncation
            max_input_tokens = 42000
            tokens = self.tokenizer.encode(str(markdown))

            if len(tokens) > max_input_tokens:
                markdown = self.tokenizer.decode(tokens[:max_input_tokens])
                markdown += "\n\n[SYSTEM WARNING: Document was truncated due to length limits. Extract relevant data from the available text above.]"

            # 3. Pass to Scout LLM for optimization
            scout_payload = {
                "messages": [
                    {
                        "role": "system",
                        "content": "You are a technical data extraction scout. Read the following website markdown and extract ONLY the information relevant to the user's query context. Be concise, retain all technical accuracy, code blocks, and configurations. If the answer is not in the text, explicitly state that.",
                    },
                    {
                        "role": "user",
                        "content": f"Query Context: {query_context}\n\nWebsite Content:\n{markdown}",
                    },
                ],
                "temperature": 0.1,
                "max_tokens": 4096,
            }

            scout_resp = requests.post(
                self.scout_api_url, json=scout_payload, timeout=120
            )
            scout_resp.raise_for_status()
            scout_data = scout_resp.json()

            optimized_text = scout_data["choices"][0]["message"]["content"]
            return f"--- Scout Data Extracted from {url} ---\n{optimized_text}"

        except Exception as e:
            return f"Tool execution error: {str(e)}"