SpiderSite Complete Guide

┌─────────────────────────────────────────────────────────────────────────┐
│                         SpiderSite Flow                                  │
├─────────────────────────────────────────────────────────────────────────┤
│                                                                          │
│  1. CRAWL PHASE                                                          │
│     ├── Check for sitemap.xml (fastest method)                          │
│     ├── Score URLs by relevance (contact, about, team pages first)      │
│     ├── Auto-detect SPA (React/Vue/Angular) → use Playwright            │
│     └── Crawl up to max_pages using selected strategy                   │
│                                                                          │
│  2. EXTRACTION PHASE (No AI - Always runs)                              │
│     ├── Extract emails, phones, addresses                               │
│     ├── Find social media profiles (14 platforms)                       │
│     └── Generate markdown compendium of all content                     │
│                                                                          │
│  3. AI ANALYSIS PHASE (Opt-in - ONE unified call)                       │
│     └── Combines ALL enabled features:                                   │
│         ├── extract_team → Team members with titles/emails              │
│         ├── extract_company_info → Company summary/services             │
│         ├── extract_pain_points → Business challenges                   │
│         ├── Lead scoring (CHAMP) → If product/ICP provided              │
│         └── custom_ai_prompt → Your custom analysis                     │
│                                                                          │
│  4. RESPONSE                                                             │
│     └── Structured JSON with all extracted data                         │
│                                                                          │
└─────────────────────────────────────────────────────────────────────────┘

{
  "data": {
    "custom_analysis": {
      "security_certifications": ["SOC 2 Type II", "ISO 27001"],
      "compliance_frameworks": ["GDPR", "CCPA", "HIPAA"],
      "data_privacy_summary": "Company maintains strict data encryption..."
    }
  }
}

{
  "success": true,
  "job_id": "uuid",
  "type": "spiderSite",
  "status": "completed",
  "processing_time_seconds": 25.4,
  "data": {
    "url": "https://example.com",
    "pages_crawled": 10,
    "crawl_status": "success",

    "emails": ["contact@example.com", "sales@example.com"],
    "phones": ["+1-555-123-4567"],
    "addresses": ["123 Main St, SF, CA"],

    "linkedin": "https://linkedin.com/company/example",
    "twitter": "https://twitter.com/example",
    "facebook": null,
    "instagram": null,
    "youtube": null,
    "tiktok": null,
    "github": "https://github.com/example",
    "pinterest": null,
    "snapchat": null,
    "reddit": null,
    "medium": null,
    "discord": null,
    "whatsapp": null,
    "telegram": null,

    "company_vitals": {
      "one_sentence_summary": "...",
      "key_services": ["Service A", "Service B"],
      "target_audience": "...",
      "industry": "B2B SaaS"
    },

    "team_members": [
      {
        "name": "John Doe",
        "title": "CEO",
        "email": "john@example.com",
        "linkedin": "https://linkedin.com/in/johndoe"
      }
    ],

    "pain_points": {
      "inferred_challenges": ["Challenge 1", "Challenge 2"],
      "recent_mentions": ["News item 1"]
    },

    "lead_scoring": {
      "icp_fit_grade": "A",
      "engagement_score": 85,
      "lead_priority": "Hot",
      "champ_breakdown": {
        "challenges": "...",
        "authority": "...",
        "money": "...",
        "prioritization": "..."
      }
    },

    "custom_analysis": {
      "your_custom_fields": "..."
    },

    "markdown_compendium": "# Company Name\n\n...",

    "compendium": {
      "available": true,
      "storage_location": "inline",
      "size_chars": 45000,
      "cleanup_level": "fit"
    },

    "metadata": {
      "crawl_strategy": "sitemap",
      "spa_enabled": true,
      "browser_rendering_available": true
    }
  }
}

{
  "data": {
    "markdown_compendium": null,
    "compendium": {
      "storage_location": "r2",
      "r2_url": "https://cdn.spideriq.di-atomic.com/compendiums/abc123.md?...",
      "url_expires_at": "2025-10-28T12:00:00Z"
    }
  }
}

import requests
import time

# Configuration
API_BASE = "https://spideriq.di-atomic.com/api/v1"
CLIENT_TOKEN = "<your_client_id>:<your_api_key>:<your_api_secret>"
headers = {"Authorization": f"Bearer {CLIENT_TOKEN}"}

# Step 1: Submit job
submit_data = {
    "payload": {
        "url": "https://target-company.com",
        "max_pages": 10,
        "extract_company_info": True,
        "extract_team": True
    }
}

response = requests.post(
    f"{API_BASE}/jobs/spiderSite/submit",
    headers=headers,
    json=submit_data
)
job_id = response.json()['job_id']
print(f"✓ Job submitted: {job_id}")

# Step 2: Poll for completion
max_wait = 120  # 2 minutes
start_time = time.time()

while time.time() - start_time < max_wait:
    response = requests.get(
        f"{API_BASE}/jobs/{job_id}/results",
        headers=headers
    )

    result = response.json()

    if result['status'] == 'completed':
        print("✓ Job completed!")
        data = result['data']

        # Access extracted data
        print(f"\nEmails: {data['emails']}")
        print(f"Phones: {data['phones']}")
        print(f"LinkedIn: {data['linkedin']}")

        if data.get('company_vitals'):
            print(f"\nCompany: {data['company_vitals']['one_sentence_summary']}")

        if data.get('team_members'):
            print(f"\nTeam Members: {len(data['team_members'])}")
            for member in data['team_members']:
                print(f"  - {member['name']}: {member.get('title', 'N/A')}")

        break

    elif result['status'] == 'failed':
        print(f"✗ Job failed: {result.get('error_message')}")
        break

    else:
        print(f"⏳ Status: {result['status']}...")
        time.sleep(3)

else:
    print("✗ Timeout waiting for job to complete")