// Use computer wait action between page loads
await mcp__claude-in-chrome__computer({
  action: "wait",
  duration: 2,  // seconds
  tabId: tabId
});

async function preflightCheck() {
  const checks = {
    sequential_thinking: false,
    claude_in_chrome: false,
    memory_mcp: false
  };

  // Check sequential-thinking MCP (required)
  try {
    await mcp__sequential-thinking__sequentialthinking({
      thought: "Preflight check - verifying MCP availability for web scraping",
      thoughtNumber: 1,
      totalThoughts: 1,
      nextThoughtNeeded: false
    });
    checks.sequential_thinking = true;
  } catch (error) {
    console.error("Sequential-thinking MCP unavailable:", error);
    throw new Error("CRITICAL: sequential-thinking MCP required but unavailable");
  }

  // Check claude-in-chrome MCP (required)
  try {
    const context = await mcp__claude-in-chrome__tabs_context_mcp({});
    checks.claude_in_chrome = true;
  } catch (error) {
    console.error("Claude-in-chrome MCP unavailable:", error);
    throw new Error("CRITICAL: claude-in-chrome MCP required but unavailable");
  }

  // Check memory-mcp (optional but recommended)
  try {
    checks.memory_mcp = true;
  } catch (error) {
    console.warn("Memory MCP unavailable - extracted data will not be persisted");
    checks.memory_mcp = false;
  }

  return checks;
}

async function extractWithRetry(extractor, context, maxRetries = 3) {
  let lastError = null;

  for (let attempt = 1; attempt <= maxRetries; attempt++) {
    try {
      const data = await extractor(context);
      return data;
    } catch (error) {
      lastError = error;
      console.error(`Extraction attempt ${attempt} failed:`, error.message);

      if (isRateLimitError(error)) {
        const delay = Math.pow(2, attempt) * 5; // 10s, 20s, 40s
        await sleep(delay * 1000);
      } else if (!isRecoverableError(error)) {
        break;
      }
    }
  }
  throw lastError;
}

function isRecoverableError(error) {
  const nonRecoverable = [
    "CRITICAL: sequential-thinking MCP required",
    "CRITICAL: claude-in-chrome MCP required",
    "Access denied",
    "CAPTCHA",
    "Blocked"
  ];
  return !nonRecoverable.some(msg => error.message.includes(msg));
}

const VALIDATION_RULES = {
  required_fields: ["name", "url"],  // Must be present
  field_types: {
    price: "number",
    name: "string",
    url: "url",
    date: "date"
  },
  constraints: {
    price: { min: 0, max: 1000000 },
    name: { minLength: 1, maxLength: 500 },
    url: { pattern: /^https?:\/\// }
  }
};

function validateRecord(record, rules) {
  const errors = [];

  // Check required fields
  for (const field of rules.required_fields) {
    if (!record[field]) {
      errors.push(`Missing required field: ${field}`);
    }
  }

  // Check field types
  for (const [field, type] of Object.entries(rules.field_types)) {
    if (record[field] && !isValidType(record[field], type)) {
      errors.push(`Invalid type for ${field}: expected ${type}`);
    }
  }

  return { valid: errors.length === 0, errors };
}

extraction_plan:
  target_url: string
  output_format: "json" | "csv" | "markdown"
  schema:
    fields:
      - name: string
        selector: string
        type: string | number | url | date
        required: boolean
  pagination:
    type: "numbered" | "next_button" | "infinite_scroll" | "load_more" | "none"
    max_pages: number
    delay_seconds: number
  rate_limit:
    delay_between_requests: number

// 1. Get existing context
const context = await mcp__claude-in-chrome__tabs_context_mcp({});

// 2. Create dedicated tab
const newTab = await mcp__claude-in-chrome__tabs_create_mcp({});
const tabId = newTab.tabId;

// 3. Navigate to target
await mcp__claude-in-chrome__navigate({
  url: targetUrl,
  tabId: tabId
});

// 4. Wait for page load
await mcp__claude-in-chrome__computer({
  action: "wait",
  duration: 2,
  tabId: tabId
});

// 5. Screenshot initial state
await mcp__claude-in-chrome__computer({
  action: "screenshot",
  tabId: tabId
});

// Read page structure
const pageTree = await mcp__claude-in-chrome__read_page({
  tabId: tabId,
  filter: "all"  // Get all elements for analysis
});

// For interactive elements only
const interactive = await mcp__claude-in-chrome__read_page({
  tabId: tabId,
  filter: "interactive"  // Buttons, links, inputs
});

// Find specific elements
const products = await mcp__claude-in-chrome__find({
  query: "product cards or listing items",
  tabId: tabId
});

const prices = await mcp__claude-in-chrome__find({
  query: "price elements",
  tabId: tabId
});

const text = await mcp__claude-in-chrome__get_page_text({
  tabId: tabId
});

const data = await mcp__claude-in-chrome__javascript_tool({
  action: "javascript_exec",
  tabId: tabId,
  text: `
    // Extract product data
    Array.from(document.querySelectorAll('.product-card')).map(card => ({
      name: card.querySelector('.product-title')?.textContent?.trim(),
      price: parseFloat(card.querySelector('.price')?.textContent?.replace(/[^0-9.]/g, '')),
      url: card.querySelector('a')?.href,
      image: card.querySelector('img')?.src
    }))
  `
});

const tableData = await mcp__claude-in-chrome__javascript_tool({
  action: "javascript_exec",
  tabId: tabId,
  text: `
    // Extract table data
    const table = document.querySelector('table');
    const headers = Array.from(table.querySelectorAll('th')).map(th => th.textContent.trim());
    const rows = Array.from(table.querySelectorAll('tbody tr')).map(tr => {
      const cells = Array.from(tr.querySelectorAll('td')).map(td => td.textContent.trim());
      return headers.reduce((obj, header, i) => ({ ...obj, [header]: cells[i] }), {});
    });
    rows
  `
});

// Scroll to load content
await mcp__claude-in-chrome__computer({
  action: "scroll",
  scroll_direction: "down",
  scroll_amount: 5,
  tabId: tabId,
  coordinate: [500, 500]
});

// Wait for content to load
await mcp__claude-in-chrome__computer({
  action: "wait",
  duration: 2,
  tabId: tabId
});

// Now extract
const data = await extractData(tabId);

async function extractNumberedPages(baseUrl, maxPages, tabId) {
  const allData = [];

  for (let page = 1; page <= maxPages; page++) {
    const url = `${baseUrl}?page=${page}`;
    await mcp__claude-in-chrome__navigate({ url, tabId });
    await mcp__claude-in-chrome__computer({ action: "wait", duration: 2, tabId });

    const pageData = await extractPageData(tabId);
    if (pageData.length === 0) break;  // No more data

    allData.push(...pageData);

    // Rate limiting
    await mcp__claude-in-chrome__computer({ action: "wait", duration: 2, tabId });
  }

  return allData;
}

async function extractWithNextButton(tabId) {
  const allData = [];
  let hasNext = true;

  while (hasNext) {
    const pageData = await extractPageData(tabId);
    allData.push(...pageData);

    // Find next button
    const nextButton = await mcp__claude-in-chrome__find({
      query: "next page button or pagination next link",
      tabId: tabId
    });

    if (nextButton && nextButton.elements && nextButton.elements.length > 0) {
      await mcp__claude-in-chrome__computer({
        action: "left_click",
        ref: nextButton.elements[0].ref,
        tabId: tabId
      });
      await mcp__claude-in-chrome__computer({ action: "wait", duration: 2, tabId });
    } else {
      hasNext = false;  // No more pages
    }
  }

  return allData;
}

async function extractInfiniteScroll(tabId, maxScrolls = 20) {
  const allData = [];
  let previousCount = 0;
  let scrollCount = 0;

  while (scrollCount < maxScrolls) {
    const pageData = await extractPageData(tabId);

    if (pageData.length === previousCount) {
      // No new content loaded, done
      break;
    }

    allData.length = 0;
    allData.push(...pageData);
    previousCount = pageData.length;

    // Scroll down
    await mcp__claude-in-chrome__computer({
      action: "scroll",
      scroll_direction: "down",
      scroll_amount: 5,
      tabId: tabId,
      coordinate: [500, 500]
    });

    await mcp__claude-in-chrome__computer({ action: "wait", duration: 2, tabId });
    scrollCount++;
  }

  return allData;
}

function toJSON(data, pretty = true) {
  return pretty ? JSON.stringify(data, null, 2) : JSON.stringify(data);
}

function toCSV(data) {
  if (data.length === 0) return "";

  const headers = Object.keys(data[0]);
  const headerRow = headers.join(",");

  const dataRows = data.map(row =>
    headers.map(h => {
      const val = row[h];
      // Escape quotes and wrap in quotes if contains comma
      if (typeof val === "string" && (val.includes(",") || val.includes('"'))) {
        return `"${val.replace(/"/g, '""')}"`;
      }
      return val;
    }).join(",")
  );

  return [headerRow, ...dataRows].join("\n");
}

function toMarkdownTable(data) {
  if (data.length === 0) return "";

  const headers = Object.keys(data[0]);
  const headerRow = `| ${headers.join(" | ")} |`;
  const separator = `| ${headers.map(() => "---").join(" | ")} |`;

  const dataRows = data.map(row =>
    `| ${headers.map(h => String(row[h] || "")).join(" | ")} |`
  );

  return [headerRow, separator, ...dataRows].join("\n");
}

// Store in Memory MCP
await memory_store({
  namespace: `skills/tooling/web-scraping/${projectName}/${timestamp}`,
  data: {
    extraction_metadata: {
      source_url: targetUrl,
      extraction_date: new Date().toISOString(),
      record_count: data.length,
      output_format: outputFormat,
      pages_scraped: pageCount
    },
    extracted_data: data
  },
  tags: {
    WHO: `web-scraping-${sessionId}`,
    WHEN: new Date().toISOString(),
    PROJECT: projectName,
    WHY: "data-extraction",
    data_type: dataType,
    record_count: data.length
  }
});

{
  "WHO": "web-scraping-{session_id}",
  "WHEN": "ISO8601_timestamp",
  "PROJECT": "{project_name}",
  "WHY": "data-extraction",
  "source_domain": "example.com",
  "data_type": "products|articles|listings|tables",
  "record_count": 150,
  "pages_scraped": 5
}

Thought 1/6: Need to extract product name, price, URL, and image from catalog
Thought 2/6: Schema: {name: string, price: number, url: url, image: url}
Thought 3/6: Identify product card containers via read_page
Thought 4/6: Use javascript_tool to query all product cards
Thought 5/6: Detect pagination pattern (numbered pages with ?page=N)
Thought 6/6: Output format: JSON array, validate price > 0

// 1. Navigate to catalog
await navigate({ url: "https://example.com/products", tabId });

// 2. Analyze structure
const structure = await read_page({ tabId, filter: "all" });

// 3. Extract data
const products = await javascript_tool({
  action: "javascript_exec",
  tabId,
  text: `
    Array.from(document.querySelectorAll('.product-card')).map(card => ({
      name: card.querySelector('.title')?.textContent?.trim(),
      price: parseFloat(card.querySelector('.price')?.textContent?.replace(/[^0-9.]/g, '')),
      url: card.querySelector('a')?.href,
      image: card.querySelector('img')?.src
    }))
  `
});

// 4. Handle pagination (repeat for each page)
// 5. Transform to JSON
const output = JSON.stringify(products, null, 2);

[
  {
    "name": "Product A",
    "price": 29.99,
    "url": "https://example.com/products/a",
    "image": "https://example.com/images/a.jpg"
  }
]

Thought 1/4: Single page extraction, no pagination needed
Thought 2/4: Schema: {headline: string, author: string, date: date, body: string}
Thought 3/4: Use get_page_text for full content, javascript_tool for metadata
Thought 4/4: Output format: Markdown for readability

// 1. Navigate to article
await navigate({ url: "https://news.example.com/article/123", tabId });

// 2. Extract metadata
const metadata = await javascript_tool({
  action: "javascript_exec",
  tabId,
  text: `({
    headline: document.querySelector('h1')?.textContent?.trim(),
    author: document.querySelector('.author')?.textContent?.trim(),
    date: document.querySelector('time')?.getAttribute('datetime')
  })`
});

// 3. Extract body text
const body = await get_page_text({ tabId });

// 4. Combine
const article = { ...metadata, body };

undefined