/**
 * Web Research Service
 *
 * Provides web search and website fetching capabilities for competitor research.
 * Uses a hybrid approach:
 *   1. First tries Ollama tool calling (for models that support web_search/web_fetch)
 *   2. Falls back to direct web search (DuckDuckGo HTML) + HTTP fetch for websites
 *   3. Falls back to AI-only generation if web research fails
 *
 * Used by the competitor pipeline to perform real web research before
 * generating competitor profiles.
 */

import { getAIConfig, APIKeyEntry } from '../../utils/aiProvider';

// ============================================
// TYPES
// ============================================

export interface WebSearchResult {
  title: string;
  url: string;
  snippet: string;
}

export interface WebsiteFetchResult {
  url: string;
  title: string;
  description: string;
  keyInfo: Record<string, string>;
  rawContent: string; // trimmed to ~8K chars
}

export interface WebResearchResult {
  searchResults: WebSearchResult[];
  websiteContent: WebsiteFetchResult | null;
  detectedWebsite: string | null;
  researchSummary: string;
}

// ============================================
// CONFIGURATION
// ============================================

const RESEARCH_TIMEOUT_MS = 90_000; // 90 seconds for entire research phase
const WEBSITE_FETCH_TIMEOUT_MS = 15_000; // 15 seconds per website fetch
const SEARCH_TIMEOUT_MS = 20_000; // 20 seconds for search
const MAX_CONTENT_LENGTH = 8000; // Trim website content to 8K chars

// ============================================
// OFFICIAL WEBSITE RESOLUTION
// ============================================

/**
 * Hosts that are never a company's own website. Search for a company and the
 * top hits are usually directories, marketplaces, news and review sites — the
 * old "first result that isn't a social network" rule happily returned those,
 * which is why competitors ended up with a Crunchbase or listicle URL as their
 * official site.
 */
const NON_OFFICIAL_HOST_PATTERNS = [
  // Social / video
  'facebook.', 'instagram.', 'twitter.', 'x.com', 'linkedin.', 'youtube.', 'youtu.be',
  'tiktok.', 'pinterest.', 'reddit.', 'threads.net', 'snapchat.',
  // Reference / encyclopaedic
  'wikipedia.', 'wikiwand.', 'wikidata.', 'fandom.',
  // Company directories / data providers
  'crunchbase.', 'pitchbook.', 'zoominfo.', 'owler.', 'similarweb.', 'apollo.io',
  'dnb.com', 'bloomberg.', 'zaubacorp.', 'tofler.', 'opencorporates.',
  // Reviews / marketplaces
  'g2.com', 'capterra.', 'trustpilot.', 'trustradius.', 'getapp.', 'softwareadvice.',
  'glassdoor.', 'indeed.', 'ambitionbox.', 'yelp.', 'tripadvisor.', 'justdial.',
  'amazon.', 'ebay.', 'flipkart.', 'alibaba.', 'etsy.', 'walmart.', 'myntra.',
  // Publishers / blogs
  'medium.com', 'substack.', 'quora.', 'blogspot.', 'wordpress.com', 'forbes.',
  'techcrunch.', 'businessinsider.', 'inc.com', 'entrepreneur.com', 'cnbc.',
  'reuters.', 'nytimes.', 'theguardian.', 'economictimes.', 'livemint.',
  'yourstory.', 'inc42.', 'hindustantimes.', 'timesofindia.',
  // Search engines / aggregators
  'google.', 'bing.com', 'duckduckgo.', 'yahoo.', 'baidu.',
];

/** Suffixes stripped when comparing a domain to a company name. */
const NAME_NOISE_WORDS = new Set([
  'inc', 'llc', 'ltd', 'limited', 'corp', 'corporation', 'co', 'company',
  'group', 'holdings', 'plc', 'gmbh', 'pvt', 'private', 'technologies',
  'technology', 'tech', 'solutions', 'systems', 'labs', 'the', 'and',
]);

function hostnameOf(url: string): string | null {
  try {
    const parsed = new URL(url.startsWith('http') ? url : `https://${url}`);
    return parsed.hostname.toLowerCase().replace(/^www\./, '');
  } catch {
    return null;
  }
}

function isNonOfficialHost(hostname: string): boolean {
  return NON_OFFICIAL_HOST_PATTERNS.some((pattern) => hostname.includes(pattern));
}

/** "AirFlex Running Shoes Inc." → ["airflex","running","shoes"] */
function nameTokens(name: string): string[] {
  return name
    .toLowerCase()
    .replace(/[^a-z0-9\s]/g, ' ')
    .split(/\s+/)
    .filter((w) => w.length > 2 && !NAME_NOISE_WORDS.has(w));
}

/**
 * Reduce a URL to the site root, e.g.
 * `https://brand.com/blog/best-shoes-2026?utm_source=x` → `https://brand.com`.
 * A competitor's "website" is the site itself, never a deep article URL.
 */
export function toSiteRoot(url: string): string | null {
  try {
    const parsed = new URL(url.startsWith('http') ? url : `https://${url}`);
    if (!parsed.hostname || !parsed.hostname.includes('.')) return null;
    const protocol = parsed.protocol === 'http:' ? 'http:' : 'https:';
    return `${protocol}//${parsed.hostname}`;
  } catch {
    return null;
  }
}

/**
 * Pick the competitor's own website from a set of search results.
 *
 * Candidates are scored rather than taken in order: a domain whose name matches
 * the company we searched for is what we want, and directory/news/marketplace
 * hosts are rejected outright. Returns the site root, or null when nothing
 * credible is found — a null website is better than a confidently wrong one.
 */
export function resolveOfficialWebsite(
  results: WebSearchResult[],
  nameHint?: string,
): string | null {
  const candidates: { url: string; score: number }[] = [];

  results.forEach((result, index) => {
    const hostname = hostnameOf(result.url);
    if (!hostname || isNonOfficialHost(hostname)) return;

    const root = toSiteRoot(result.url);
    if (!root) return;

    let score = nameHint ? domainNameAffinity(hostname, nameHint) : 0;

    // A root-level hit is more likely the homepage than a deep link.
    const path = result.url.replace(/^https?:\/\/[^/]+/i, '').replace(/[/?#]+$/, '');
    if (!path) score += 15;
    else if (path.split('/').filter(Boolean).length <= 1) score += 5;

    // Earlier results are weakly preferred, as a tiebreak only.
    score += Math.max(0, 8 - index);

    candidates.push({ url: root, score });
  });

  if (candidates.length === 0) return null;

  const best = candidates.reduce((a, b) => (b.score > a.score ? b : a));
  // Require a real signal: without a name match or a homepage-looking hit, the
  // top result is just as likely to be an unrelated article.
  return best.score >= 20 ? best.url : null;
}

/**
 * How strongly a hostname looks like it belongs to this company.
 *
 * Only the brand part of the name counts — the leading token, plus the whole
 * name de-spaced. Matching on any token would let a competitor called
 * "AirFlex Running Shoes" claim `running.com`, which is precisely the kind of
 * wrong-but-plausible URL this is meant to prevent.
 */
function domainNameAffinity(hostname: string, name: string): number {
  const tokens = nameTokens(name);
  if (tokens.length === 0) return 0;

  const joined = tokens.join('');
  const brand = tokens[0];

  // Score every label except the public suffix, so brand subdomains such as
  // `about.nike.com` or `shop.brand.co.uk` are recognised too.
  const labels = hostname.split('.');
  const suffixParts = labels.length > 2 && ['co', 'com', 'org', 'net', 'gov', 'edu', 'ac'].includes(labels[labels.length - 2])
    ? 2
    : 1;
  const nameLabels = labels.slice(0, Math.max(1, labels.length - suffixParts));

  let bestScore = 0;
  for (const rawLabel of nameLabels) {
    const label = rawLabel.replace(/[^a-z0-9]/g, '');
    if (!label) continue;

    let score = 0;
    if (label === joined || label === brand) score = 60;
    // Brands often extend their domain ("airflex" → "airflexshoes"), but a
    // short brand token would match far too much, so require some length.
    else if (brand.length >= 4 && label.includes(brand)) score = 35;
    else if (label.length >= 4 && joined.startsWith(label)) score = 30;

    if (score > bestScore) bestScore = score;
  }

  return bestScore;
}

/**
 * Does this URL's domain plausibly belong to the named company?
 * Used to decide whether a site found while researching the market at large
 * can be claimed as a specific competitor's official website.
 */
export function domainMatchesCompanyName(url: string, name: string): boolean {
  const hostname = hostnameOf(url);
  if (!hostname || !name) return false;
  return domainNameAffinity(hostname, name) >= 30;
}

/**
 * Look up a single company's official website.
 *
 * Auto-Fill invents the competitor during generation, so stage 0 research (run
 * before the name exists) can't have looked for its site. This targeted search
 * runs once the name is known, which is the only way an invented-but-real
 * competitor gets a URL that actually resolves.
 *
 * Returns null when nothing credible is found — never a guess.
 */
export async function findOfficialWebsite(companyName: string): Promise<string | null> {
  const name = companyName?.trim();
  if (!name) return null;

  try {
    const results = await searchWeb(`${name} official website`);
    if (results.length === 0) return null;

    const resolved = resolveOfficialWebsite(results, name);
    // Only accept a domain that actually looks like this company's — otherwise
    // we would just be relabelling the top search hit.
    if (resolved && domainMatchesCompanyName(resolved, name)) return resolved;

    console.log(`[WebResearch] No official website matched "${name}"`);
    return null;
  } catch (error: any) {
    console.warn(`[WebResearch] Official website lookup failed for "${name}": ${error.message}`);
    return null;
  }
}

// ============================================
// TOOL DEFINITIONS (for Ollama tool calling)
// ============================================

const WEB_TOOLS = [
  {
    type: 'function' as const,
    function: {
      name: 'web_search',
      description: 'Search the web for information about a company, product, or topic. Returns a list of search results with titles, URLs, and snippets.',
      parameters: {
        type: 'object',
        properties: {
          query: {
            type: 'string',
            description: 'The search query string',
          },
        },
        required: ['query'],
      },
    },
  },
  {
    type: 'function' as const,
    function: {
      name: 'web_fetch',
      description: "Fetch the content of a web page by URL. Returns the page's text content for analysis.",
      parameters: {
        type: 'object',
        properties: {
          url: {
            type: 'string',
            description: 'The URL of the web page to fetch',
          },
        },
        required: ['url'],
      },
    },
  },
];

// ============================================
// MAIN EXPORT: PERFORM WEB RESEARCH
// ============================================

/**
 * Perform web research using Ollama tool calling (primary) or
 * direct web search/fetch (fallback).
 *
 * Returns null if all approaches fail (graceful degradation).
 */
export async function performWebResearch(
  query: string,
  websiteUrl?: string,
  onProgress?: (step: string) => void,
  /** Company name, used to recognise its own domain among the search results. */
  nameHint?: string,
): Promise<WebResearchResult | null> {
  console.log(`[WebResearch] Starting web research for: "${query}"${websiteUrl ? `, website: ${websiteUrl}` : ''}`);

  // Strategy 1: Direct web search + fetch (most reliable)
  try {
    const directResult = await performDirectWebResearch(query, websiteUrl, onProgress, nameHint);
    if (directResult && (directResult.searchResults.length > 0 || directResult.websiteContent || directResult.detectedWebsite)) {
      console.log(`[WebResearch] Direct research succeeded. Website: ${directResult.detectedWebsite || 'none'}`);
      return directResult;
    }
  } catch (error: any) {
    console.warn(`[WebResearch] Direct research failed: ${error.message}`);
  }

  // Strategy 2: Ollama tool calling (for models that support web_search/web_fetch)
  try {
    const toolResult = await performOllamaToolResearch(query, websiteUrl, onProgress);
    if (toolResult && (toolResult.searchResults.length > 0 || toolResult.websiteContent || toolResult.detectedWebsite)) {
      console.log(`[WebResearch] Ollama tool research succeeded. Website: ${toolResult.detectedWebsite || 'none'}`);
      return toolResult;
    }
  } catch (error: any) {
    console.warn(`[WebResearch] Ollama tool research failed: ${error.message}`);
  }

  // Strategy 3: AI knowledge (no real web research)
  console.log('[WebResearch] All web research strategies failed — returning null for AI-only generation');
  return null;
}

// ============================================
// STRATEGY 1: DIRECT WEB RESEARCH
// ============================================

/**
 * Direct web research using DuckDuckGo HTML search + HTTP fetch for websites.
 * This is the most reliable approach since it doesn't depend on model tool calling.
 */
async function performDirectWebResearch(
  query: string,
  websiteUrl?: string,
  onProgress?: (step: string) => void,
  nameHint?: string,
): Promise<WebResearchResult | null> {
  const searchResults: WebSearchResult[] = [];
  let websiteContent: WebsiteFetchResult | null = null;
  // A caller-supplied website wins, but is still reduced to its root.
  let detectedWebsite: string | null = websiteUrl ? toSiteRoot(websiteUrl) : null;

  // Step 1: Search for competitor information
  onProgress?.('Searching the web for competitor information...');
  try {
    const results = await searchWeb(query);
    searchResults.push(...results);
    console.log(`[WebResearch] Found ${results.length} search results`);

    // Identify the company's own site among the results. This deliberately
    // returns null rather than guessing: an unrelated top result presented as
    // the competitor's official website is worse than no website at all.
    if (!detectedWebsite && results.length > 0) {
      detectedWebsite = resolveOfficialWebsite(results, nameHint);
      if (detectedWebsite) {
        console.log(`[WebResearch] Official website resolved: ${detectedWebsite}${nameHint ? ` (matched "${nameHint}")` : ''}`);
      } else {
        console.log('[WebResearch] No credible official website among search results');
      }
    }
  } catch (error: any) {
    console.warn(`[WebResearch] Web search failed: ${error.message}`);
  }

  // Step 2: Fetch the competitor's website for detailed info
  if (detectedWebsite) {
    onProgress?.(`Fetching website: ${detectedWebsite}...`);
    try {
      const fetchedContent = await fetchWebsite(detectedWebsite);
      if (fetchedContent) {
        websiteContent = fetchedContent;
      }
    } catch (error: any) {
      console.warn(`[WebResearch] Website fetch failed for ${detectedWebsite}: ${error.message}`);
    }
  }

  // Step 3: Use AI to summarize and structure the research data
  let researchSummary = '';
  if (searchResults.length > 0 || websiteContent) {
    onProgress?.('Analyzing research data...');
    researchSummary = await summarizeResearch(query, searchResults, websiteContent);
  }

  if (searchResults.length === 0 && !websiteContent && !researchSummary) {
    return null;
  }

  return {
    searchResults,
    websiteContent,
    detectedWebsite,
    researchSummary,
  };
}

// ============================================
// STRATEGY 2: OLLAMA TOOL CALLING
// ============================================

/**
 * Research using Ollama's tool calling API (for models that support web_search/web_fetch).
 * This approach lets the model decide what to search for and which pages to fetch.
 */
async function performOllamaToolResearch(
  query: string,
  websiteUrl?: string,
  onProgress?: (step: string) => void,
): Promise<WebResearchResult | null> {
  const config = await getAIConfig();
  const ollamaUrl = config.OLLAMA_BASE_URL || process.env.OLLAMA_BASE_URL || 'http://localhost:11434';
  const isLocalOllama = ollamaUrl.includes('localhost') || ollamaUrl.includes('127.0.0.1');

  let ollamaModel: string;
  let ollamaKey: string;

  if (isLocalOllama) {
    ollamaModel = process.env.OLLAMA_LOCAL_MODEL || 'qwen2.5:7b';
    ollamaKey = '';
  } else {
    const keyList: APIKeyEntry[] = config.OLLAMA_KEY_LIST || [];
    const activeKey = keyList.find(k => k.isActive !== false && k.health !== 'inactive');
    const fallbackKey = keyList[0];

    if (!activeKey && !fallbackKey) {
      console.log('[WebResearch] No Ollama API keys available for tool calling');
      return null;
    }

    const usedKey = activeKey || fallbackKey;
    ollamaKey = usedKey.key;
    ollamaModel = usedKey.model || config.OLLAMA_MODEL || process.env.OLLAMA_MODEL || 'qwen2.5:7b';
  }

  const headers: Record<string, string> = { 'Content-Type': 'application/json' };
  if (ollamaKey) {
    headers['Authorization'] = `Bearer ${ollamaKey}`;
  }

  const systemPrompt = `You are a competitive intelligence research assistant with access to web_search and web_fetch tools.

Research the given company/competitor thoroughly:
1. Use web_search to find information about them
2. If you find their website URL, use web_fetch to get detailed information from it
3. Summarize all findings in a structured format

IMPORTANT: Respond with ONLY valid JSON when you have completed your research:
{
  "website": "the real website URL or null",
  "companyInfo": {
    "name": "company name",
    "description": "brief description",
    "foundedYear": "year or null",
    "headquarters": "location or null",
    "employeeCount": "count or null",
    "revenueEstimate": "estimate or null"
  },
  "products": ["product list"],
  "pricing": "pricing info or null",
  "marketingChannels": ["channel list"],
  "strengths": ["strength list"],
  "weaknesses": ["weakness list"],
  "searchResults": [{"title": "", "url": "", "snippet": ""}]
}`;

  const userPrompt = websiteUrl
    ? `Research this company/competitor: ${query}\nTheir website is: ${websiteUrl}\n\nUse web_search to find more information and web_fetch to get details from their website.`
    : `Research this company/competitor: ${query}\n\nUse web_search to find their website and information about them. If you find a website, use web_fetch to get details.`;

  const controller = new AbortController();
  const timeoutId = setTimeout(() => controller.abort(), RESEARCH_TIMEOUT_MS);

  try {
    onProgress?.('Researching with AI web tools...');

    const response = await fetch(`${ollamaUrl}/api/chat`, {
      method: 'POST',
      headers,
      body: JSON.stringify({
        model: ollamaModel,
        messages: [
          { role: 'system', content: systemPrompt },
          { role: 'user', content: userPrompt },
        ],
        tools: WEB_TOOLS,
        stream: false,
        options: {
          num_ctx: 32768,
          num_predict: 8000,
        },
      }),
      signal: controller.signal,
    });

    clearTimeout(timeoutId);

    if (!response.ok) {
      console.warn(`[WebResearch] Ollama tool API error (${response.status})`);
      return null;
    }

    const data = await response.json() as any;
    const message = data.message || {};

    // Handle multi-turn tool calling
    let content = message.content || '';
    const toolCalls = message.tool_calls || [];
    const messages: any[] = [
      { role: 'system', content: systemPrompt },
      { role: 'user', content: userPrompt },
    ];

    if (toolCalls.length > 0) {
      // Model wants to call tools
      messages.push({
        role: 'assistant',
        content: content,
        tool_calls: toolCalls,
      });

      // Execute each tool call
      for (const toolCall of toolCalls) {
        const toolName = toolCall.function?.name || toolCall.name;
        const toolArgs = toolCall.function?.arguments || toolCall.arguments || {};

        let toolResult: string;

        if (toolName === 'web_search') {
          const searchQuery = toolArgs.query || query;
          onProgress?.(`Searching: "${searchQuery}"...`);
          const results = await searchWeb(searchQuery);
          toolResult = JSON.stringify(results.map(r => ({ title: r.title, url: r.url, snippet: r.snippet })));
        } else if (toolName === 'web_fetch') {
          const fetchUrl = toolArgs.url || '';
          onProgress?.(`Fetching: ${fetchUrl}...`);
          const fetched = fetchUrl ? await fetchWebsite(fetchUrl) : null;
          toolResult = fetched ? JSON.stringify({ title: fetched.title, description: fetched.description, content: fetched.rawContent.substring(0, 4000) }) : '{}';
        } else {
          toolResult = JSON.stringify({ error: `Unknown tool: ${toolName}` });
        }

        messages.push({
          role: 'tool',
          content: toolResult,
          tool_call_id: toolCall.id || `${toolName}_0`,
        });
      }

      // Send tool results back to the model
      const finalResponse = await fetch(`${ollamaUrl}/api/chat`, {
        method: 'POST',
        headers,
        body: JSON.stringify({
          model: ollamaModel,
          messages,
          stream: false,
          options: {
            num_ctx: 32768,
            num_predict: 8000,
          },
        }),
        signal: controller.signal,
      });

      if (finalResponse.ok) {
        const finalData = await finalResponse.json() as any;
        content = finalData.message?.content || content;
      }
    }

    if (!content || content.trim().length < 10) {
      // Try thinking field for reasoning models
      const thinking = message.thinking || data.thinking || '';
      if (thinking && thinking.trim().length > 10) {
        content = thinking;
      }
    }

    if (!content || content.trim().length < 10) {
      return null;
    }

    // Parse the structured response
    let parsed: any = null;
    try {
      const jsonMatch = content.match(/\{[\s\S]*\}/);
      if (jsonMatch) {
        parsed = JSON.parse(jsonMatch[0]);
      }
    } catch {
      // Not valid JSON — use as raw research summary
    }

    // Normalise to the site root, and drop a host that is plainly a directory
    // or news site rather than the company's own domain.
    const rawWebsite = parsed?.website || websiteUrl || null;
    const rootWebsite = rawWebsite ? toSiteRoot(String(rawWebsite)) : null;
    const rootHost = rootWebsite ? hostnameOf(rootWebsite) : null;
    const detectedWebsite = rootHost && !isNonOfficialHost(rootHost) ? rootWebsite : null;

    return {
      searchResults: parsed?.searchResults || [],
      websiteContent: parsed?.websiteContent || (detectedWebsite ? {
        url: detectedWebsite,
        title: parsed?.companyInfo?.name || '',
        description: parsed?.companyInfo?.description || '',
        keyInfo: {
          ...(parsed?.companyInfo?.foundedYear ? { founded: String(parsed.companyInfo.foundedYear) } : {}),
          ...(parsed?.companyInfo?.headquarters ? { headquarters: parsed.companyInfo.headquarters } : {}),
        },
        rawContent: content.substring(0, MAX_CONTENT_LENGTH),
      } : null),
      detectedWebsite,
      researchSummary: content,
    };

  } catch (error: any) {
    if (error.name === 'AbortError') {
      console.log('[WebResearch] Ollama tool research timed out');
    } else {
      console.warn(`[WebResearch] Ollama tool research error: ${error.message}`);
    }
    return null;
  }
}

// ============================================
// WEB SEARCH (DuckDuckGo HTML)
// ============================================

/**
 * Search the web using DuckDuckGo HTML search.
 * Returns up to 8 search results with titles, URLs, and snippets.
 */
async function searchWeb(query: string): Promise<WebSearchResult[]> {
  const results: WebSearchResult[] = [];

  try {
    const url = `https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`;
    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), SEARCH_TIMEOUT_MS);

    const response = await fetch(url, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml',
        'Accept-Language': 'en-US,en;q=0.9',
      },
      signal: controller.signal,
    });

    clearTimeout(timeoutId);

    if (!response.ok) {
      console.warn(`[WebResearch] DuckDuckGo search returned ${response.status}`);
      return results;
    }

    const html = await response.text();

    // Parse DuckDuckGo HTML results
    // Result blocks are in <div class="result results_links results_links_deep web-result">
    const resultPattern = /<a[^>]+class="result__a"[^>]+href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/gi;
    const snippetPattern = /<a[^>]+class="result__snippet"[^>]*>([\s\S]*?)<\/a>/gi;

    const urls: string[] = [];
    const titles: string[] = [];
    let match;

    while ((match = resultPattern.exec(html)) !== null && urls.length < 8) {
      let resultUrl = match[1];
      const title = match[2].replace(/<[^>]+>/g, '').trim();

      // DuckDuckGo wraps URLs in redirect — extract the actual URL
      const uddgMatch = resultUrl.match(/uddg=([^&]+)/);
      if (uddgMatch) {
        resultUrl = decodeURIComponent(uddgMatch[1]);
      }

      // Skip non-http URLs and duplicate domains
      if (!resultUrl.startsWith('http')) continue;
      if (urls.some(u => new URL(u).hostname === new URL(resultUrl).hostname)) continue;

      urls.push(resultUrl);
      titles.push(title);
    }

    // Extract snippets
    const snippets: string[] = [];
    while ((match = snippetPattern.exec(html)) !== null && snippets.length < 8) {
      snippets.push(match[1].replace(/<[^>]+>/g, '').trim());
    }

    for (let i = 0; i < Math.min(urls.length, 8); i++) {
      results.push({
        title: titles[i] || '',
        url: urls[i],
        snippet: snippets[i] || '',
      });
    }

  } catch (error: any) {
    console.warn(`[WebResearch] DuckDuckGo search error: ${error.message}`);
  }

  return results;
}

// ============================================
// WEBSITE FETCH
// ============================================

/**
 * Fetch a website and extract its text content.
 * Returns structured info (title, description, text) or null on failure.
 */
async function fetchWebsite(url: string): Promise<WebsiteFetchResult | null> {
  try {
    // Ensure URL has protocol
    const fetchUrl = url.startsWith('http') ? url : `https://${url}`;

    const controller = new AbortController();
    const timeoutId = setTimeout(() => controller.abort(), WEBSITE_FETCH_TIMEOUT_MS);

    const response = await fetch(fetchUrl, {
      headers: {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml',
        'Accept-Language': 'en-US,en;q=0.9',
      },
      signal: controller.signal,
    });

    clearTimeout(timeoutId);

    if (!response.ok) {
      console.warn(`[WebResearch] Website fetch returned ${response.status} for ${fetchUrl}`);
      return null;
    }

    const html = await response.text();

    // Extract title
    const titleMatch = html.match(/<title[^>]*>([\s\S]*?)<\/title>/i);
    const title = titleMatch ? titleMatch[1].trim() : '';

    // Extract meta description
    const descMatch = html.match(/<meta[^>]+name=["']description["'][^>]+content=["']([^"']*)["']/i)
      || html.match(/<meta[^>]+content=["']([^"']*)["'][^>]+name=["']description["']/i);
    const description = descMatch ? descMatch[1].trim() : '';

    // Strip HTML tags to get plain text
    let text = html
      .replace(/<script[\s\S]*?<\/script>/gi, '')
      .replace(/<style[\s\S]*?<\/style>/gi, '')
      .replace(/<nav[\s\S]*?<\/nav>/gi, '')
      .replace(/<footer[\s\S]*?<\/footer>/gi, '')
      .replace(/<header[\s\S]*?<\/header>/gi, '')
      .replace(/<[^>]+>/g, ' ')
      .replace(/\s+/g, ' ')
      .trim();

    // Limit content length
    text = text.substring(0, MAX_CONTENT_LENGTH);

    // Try to extract key info from the text
    const keyInfo: Record<string, string> = {};

    // Try to find founding year
    const foundedMatch = text.match(/(?:founded|established|started|incorporated)\s+(?:in\s+)?(\d{4})/i);
    if (foundedMatch) keyInfo.founded = foundedMatch[1];

    // Try to find headquarters
    const hqMatch = text.match(/(?:headquarters|HQ|based in|located in)\s+([A-Z][a-zA-Z\s]+(?:,\s*[A-Z][a-zA-Z\s]+)?)/);
    if (hqMatch) keyInfo.headquarters = hqMatch[1].trim();

    // Try to find employee count
    const empMatch = text.match(/(\d[\d,+]*)\+?\s*(?:employees|team members|people|staff)/i);
    if (empMatch) keyInfo.employees = empMatch[1];

    // Try to find revenue
    const revMatch = text.match(/\$[\d.]+\s*(?:million|billion|M|B)\s*(?:in\s+)?(?:revenue|ARR|annual\s+revenue)/i);
    if (revMatch) keyInfo.revenue = revMatch[0];

    return {
      url: fetchUrl,
      title,
      description,
      keyInfo,
      rawContent: text,
    };

  } catch (error: any) {
    console.warn(`[WebResearch] Website fetch error for ${url}: ${error.message}`);
    return null;
  }
}

// ============================================
// AI SUMMARIZATION
// ============================================

/**
 * Use AI to summarize and structure the web research data.
 * This produces a clean, structured summary that can be included in the
 * competitor generation prompts.
 */
async function summarizeResearch(
  query: string,
  searchResults: WebSearchResult[],
  websiteContent: WebsiteFetchResult | null,
): Promise<string> {
  try {
    const { generateWithAI } = await import('../../utils/aiProvider');

    const searchContext = searchResults.length > 0
      ? searchResults.map(r => `- ${r.title}: ${r.url}\n  ${r.snippet}`).join('\n')
      : 'No search results available.';

    const websiteContext = websiteContent
      ? `Website: ${websiteContent.url}\nTitle: ${websiteContent.title}\nDescription: ${websiteContent.description}\nKey Info: ${JSON.stringify(websiteContent.keyInfo)}\nContent (excerpt): ${websiteContent.rawContent.substring(0, 3000)}`
      : 'No website content available.';

    const systemPrompt = `You are a competitive intelligence analyst. Summarize the web research data below into a concise, structured competitor profile. Focus on FACTUAL information found in the research. If something is uncertain, say so. Format your response as a structured text summary with these sections:

1. WEBSITE: The real website URL if found
2. COMPANY OVERVIEW: Name, description, founded year, headquarters, size
3. PRODUCTS & PRICING: Main products, pricing strategy, key features
4. MARKET POSITION: Market position, target audience, competitive advantages
5. WEAKNESSES: Known weaknesses or challenges
6. MARKETING: Marketing channels, content strategy, SEO keywords`;

    const userPrompt = `Research query: "${query}"\n\n=== SEARCH RESULTS ===\n${searchContext}\n\n=== WEBSITE CONTENT ===\n${websiteContext}`;

    const result = await generateWithAI(userPrompt, systemPrompt, 4000, 0.3, 'text');

    if (result && result.content) {
      return result.content;
    }

    return '';
  } catch (error: any) {
    console.warn(`[WebResearch] AI summarization failed: ${error.message}`);
    // Return a basic summary from the raw data
    const parts: string[] = [];
    if (searchResults.length > 0) {
      parts.push('Search Results:');
      searchResults.forEach(r => parts.push(`- ${r.title}: ${r.snippet}`));
    }
    if (websiteContent) {
      parts.push(`\nWebsite (${websiteContent.url}): ${websiteContent.description}`);
    }
    return parts.join('\n');
  }
}