# Schedulala.com Robots.txt
# Updated: July 30, 2026
#
# IMPORTANT — why there are no per-bot groups below:
# Per the robots.txt standard, a crawler obeys exactly ONE group: the most
# specific matching User-agent. Named groups do NOT inherit rules from the
# "*" group (https://developers.google.com/search/docs/crawling-indexing/robots/robots_txt).
# This file previously listed ~32 AI crawlers each with a bare "Allow: /",
# which silently exempted every one of them from all the Disallow rules —
# bingbot, GPTBot, ClaudeBot and others could crawl /api/, /admin/, /auth/,
# /dashboard and the test pages.
# Since we want AI crawlers to have the SAME access as everyone else (all
# public content, none of the private paths), the "*" group below already
# does that. Do not re-add per-bot "Allow: /" groups. Only name a crawler
# here if it genuinely needs DIFFERENT rules, and if you do, repeat the full
# Disallow list inside that group.

User-agent: *

# Allow important pages
Allow: /
Allow: /pricing
Allow: /threads
Allow: /instagram
Allow: /twitter
Allow: /tiktok
Allow: /facebook
Allow: /linkedin
Allow: /bluesky
Allow: /youtube
Allow: /pinterest
Allow: /telegram

# Allow blog and tools content
Allow: /blog/
Allow: /tools/

# Allow developer platform and AI integration pages
Allow: /developers
Allow: /developers/docs
Allow: /integrations/
Allow: /agents/

# Allow localized blog content
Allow: /es/blog/
Allow: /de/blog/
Allow: /ja/blog/
Allow: /fr/blog/
Allow: /pt/blog/
Allow: /ko/blog/
Allow: /it/blog/
Allow: /nl/blog/
Allow: /zh/blog/
Allow: /ar/blog/

# Allow comparison pages
Allow: /compare/

# Disallow API endpoints
Disallow: /api/

# Disallow authentication and private app pages
Disallow: /auth/
Disallow: /admin/
Disallow: /api/auth/
Disallow: /dashboard

# Disallow private/deprecated pages
Disallow: /connect
Disallow: /platform-disclosure

# Block oauth callback (not a real page)
Disallow: /oauth/callback

# Block data deletion page (privacy policy, not searchable content)
Disallow: /data-deletion

# Note: /blog/category/ pages use page-level noindex for empty categories,
# so they don't need a robots.txt Disallow. Categories with articles should be indexable.

# Disallow test pages
Disallow: /test-*
Disallow: /welcome
Disallow: /oauth
Disallow: /debug-*
Disallow: /bluesky-fix
Disallow: /bluesky-test
Disallow: /twitter-test

# NOTE on /_next/ — deliberately NOT blocked.
# Blocking /_next/static/ blocks the CSS and JS bundles Google needs to render
# pages. Every tool widget under /tools/ is a client component, so with those
# chunks blocked Googlebot renders 130+ "free tool" pages as an unstyled page
# with a dead input. Google: "Google Search won't render JavaScript from
# blocked files" and "if the absence of these resources make the page harder
# for Google's crawler to understand the page, don't block them."
# The old justification comment here claimed the broader "Allow: /" would still
# let Google fetch them — that is false; the more specific Disallow wins.
# Static assets are not indexed as pages, so there is nothing to suppress.

# NOTE on /platforms/ — deliberately NOT blocked.
# Those URLs 301 to the canonical /{platform} pages. Blocking a redirect stops
# Google from following it, so the link signals never consolidate. The old
# comment claimed blocking "saves crawl budget"; crawl budget applies to sites
# with 1M+ pages (or 10k+ changing daily). This site has ~740 URLs.

# Sitemap
Sitemap: https://schedulala.com/sitemap.xml

# LLM-specific content files for AI crawlers
# See https://llmstxt.org for specification
# Concise overview: https://schedulala.com/llms.txt
# Full documentation: https://schedulala.com/llms-full.txt