import json
import re
from pathlib import Path

CLEANED_DIR = Path(r"c:\Users\Daniil\VS_Code\Trash\cleaned_backups")
REPORT_PATH = Path(r"c:\Users\Daniil\VS_Code\Trash\mcp_mentions_report.md")

# Strict check for MCP related to AI/LLM
MCP_PATTERN = re.compile(r'\bmcp\b|model\s+context\s+protocol|fastmcp', re.IGNORECASE)

# Keywords specifically indicating CREATION / DEVELOPMENT / TUTORIALS / ARCHITECTURE of MCP
CREATION_PATTERNS = [
    r'курс\w*',
    r'туториал\w*',
    r'гайд\w*',
    r'учебник\w*',
    r'созда(?:ть|ние|ем|вай|вать|л[а-я]*)',
    r'написа(?:ть|ние|л[а-я]*)',
    r'разработ(?:ка|ать|чик\w*|али)',
    r'сво(?:й|его|их|ем)\s+mcp',
    r'fastmcp',
    r'mcp\s+python\s+sdk',
    r'mcp\s+(?:sdk|server|tools?|integration)',
    r'репозитор\w*',
    r'шаблон\w*',
    r'template\w*',
    r'guide',
    r'tutorial',
    r'build\w*',
    r'develop\w*'
]

compiled_creation = [re.compile(p, re.IGNORECASE) for p in CREATION_PATTERNS]

all_posts = []
for file_path in sorted(CLEANED_DIR.glob("*.jsonl")):
    with open(file_path, "r", encoding="utf-8") as f:
        for line in f:
            if not line.strip():
                continue
            post = json.loads(line)
            text = post.get("text", "")
            if MCP_PATTERN.search(text):
                # Filter out obvious false positives (Minecraft, etc.)
                if re.search(r'\b(minecraft|майнкрафт|mcpe)\b', text, re.IGNORECASE):
                    continue
                all_posts.append(post)

print(f"Total AI-MCP related posts across all channels: {len(all_posts)}")

classified_posts = []

for post in all_posts:
    text = post["text"]
    lower = text.lower()
    
    # Calculate score/signals for creation
    matched_patterns = []
    for cp in compiled_creation:
        m = cp.findall(text)
        if m:
            matched_patterns.extend(m)
            
    # Category detection:
    # 1. Guides, Courses & SDK Tutorials (Teaching how to build/create MCP)
    # 2. Open-Source MCP Servers & Development Kits (Libraries/templates/implementations)
    # 3. Ecosystem & Tool Announcements (Plugins, integrations like Figma/Unity/Yandex)
    category = "general"
    
    is_tutorial = any(k in lower for k in ["курс", "учебник", "туториал", "обуч", "tutorial", "learn", "гайд", "guide", "beginners", "sdk"])
    is_create = any(k in lower for k in ["созда", "написа", "разработ", "свой mcp", "fastmcp", "build", "create"])
    has_github = any("github.com" in link.lower() for link in post.get("links", []))
    
    if (is_tutorial or is_create) and ("mcp" in lower or "model context protocol" in lower):
        if is_tutorial:
            category = "tutorials_and_courses"
        elif has_github or "репозитор" in lower or "библиотек" in lower or "плагин" in lower:
            category = "repos_and_tools"
        else:
            category = "creation_guides"
    elif "mcp" in lower:
        category = "ecosystem_integrations"

    classified_posts.append({
        "post": post,
        "category": category,
        "matched_patterns": list(set(matched_patterns))
    })

# Summary by category
cat_counts = {}
for cp in classified_posts:
    cat_counts[cp["category"]] = cat_counts.get(cp["category"], 0) + 1

print("Categories distribution:", cat_counts)

# Let's save a structured analysis JSON
with open(r"c:\Users\Daniil\VS_Code\Trash\mcp_classified.json", "w", encoding="utf-8") as f:
    json.dump(classified_posts, f, ensure_ascii=False, indent=2)

print("Saved mcp_classified.json")
