# ============================================================ # ROBOTS.TXT — Parnell Wellness / BariWiki # https://parnellwellness.com/robots.txt # # Policy: ALLOW ALL CRAWLERS # All search engines, AI crawlers, LLM training bots, # and archival services are welcome to access this site. # ============================================================ # ============================================================ # DEFAULT RULE: Allow everything for all bots # ============================================================ User-agent: * Allow: / Disallow: # ============================================================ # GOOGLE (Search, Images, Video, News, AdsBot, etc.) # ============================================================ User-agent: Googlebot Allow: / User-agent: Googlebot-Image Allow: / User-agent: Googlebot-Video Allow: / User-agent: Googlebot-News Allow: / User-agent: Google-Extended Allow: / User-agent: Google-InspectionTool Allow: / User-agent: GoogleOther Allow: / User-agent: GoogleOther-Image Allow: / User-agent: GoogleOther-Video Allow: / User-agent: Storebot-Google Allow: / # ============================================================ # BING / MICROSOFT # ============================================================ User-agent: Bingbot Allow: / User-agent: BingPreview Allow: / User-agent: MicrosoftPreview Allow: / User-agent: adidxbot Allow: / User-agent: msnbot Allow: / User-agent: msnbot-media Allow: / # ============================================================ # OPENAI / CHATGPT # ============================================================ User-agent: GPTBot Allow: / User-agent: ChatGPT-User Allow: / User-agent: OAI-SearchBot Allow: / # ============================================================ # ANTHROPIC / CLAUDE # ============================================================ User-agent: anthropic-ai Allow: / User-agent: Claude-Web Allow: / User-agent: ClaudeBot Allow: / # ============================================================ # META / FACEBOOK # ============================================================ User-agent: FacebookExternalHit Allow: / User-agent: Facebookbot Allow: / User-agent: FacebookCatalog Allow: / User-agent: Meta-ExternalAgent Allow: / User-agent: Meta-ExternalFetcher Allow: / # ============================================================ # APPLE # ============================================================ User-agent: Applebot Allow: / User-agent: Applebot-Extended Allow: / # ============================================================ # PERPLEXITY AI # ============================================================ User-agent: PerplexityBot Allow: / # ============================================================ # COHERE # ============================================================ User-agent: cohere-ai Allow: / User-agent: cohere-training Allow: / # ============================================================ # AMAZON / ALEXA # ============================================================ User-agent: Amazonbot Allow: / User-agent: ia_archiver Allow: / # ============================================================ # COMMON CRAWL (Open Dataset used by many AI models) # ============================================================ User-agent: CCBot Allow: / # ============================================================ # INTERNET ARCHIVE / WAYBACK MACHINE # ============================================================ User-agent: archive.org_bot Allow: / # ============================================================ # TWITTER / X # ============================================================ User-agent: Twitterbot Allow: / # ============================================================ # LINKEDIN # ============================================================ User-agent: LinkedInBot Allow: / # ============================================================ # PINTEREST # ============================================================ User-agent: Pinterest Allow: / User-agent: Pinterestbot Allow: / # ============================================================ # SLACK # ============================================================ User-agent: Slackbot Allow: / User-agent: Slackbot-LinkExpanding Allow: / # ============================================================ # TELEGRAM # ============================================================ User-agent: TelegramBot Allow: / # ============================================================ # DISCORD # ============================================================ User-agent: Discordbot Allow: / # ============================================================ # WHATSAPP # ============================================================ User-agent: WhatsApp Allow: / # ============================================================ # SNAPCHAT # ============================================================ User-agent: Snapchat Allow: / # ============================================================ # YANDEX (Russian Search Engine) # ============================================================ User-agent: Yandex Allow: / User-agent: YandexBot Allow: / User-agent: YandexImages Allow: / # ============================================================ # BAIDU (Chinese Search Engine) # ============================================================ User-agent: Baiduspider Allow: / User-agent: Baiduspider-image Allow: / User-agent: Baiduspider-video Allow: / # ============================================================ # NAVER (Korean Search Engine) # ============================================================ User-agent: Yeti Allow: / # ============================================================ # DUCKDUCKGO # ============================================================ User-agent: DuckDuckBot Allow: / # ============================================================ # BRAVE SEARCH # ============================================================ User-agent: BraveSearch Allow: / # ============================================================ # MOJEEK # ============================================================ User-agent: MojeekBot Allow: / # ============================================================ # QWANT (European Search Engine) # ============================================================ User-agent: Qwantify Allow: / # ============================================================ # SOGOU (Chinese Search Engine) # ============================================================ User-agent: Sogou Allow: / User-agent: sogou spider Allow: / # ============================================================ # SEZNAM (Czech Search Engine) # ============================================================ User-agent: SeznamBot Allow: / # ============================================================ # AI / LLM TRAINING & RESEARCH CRAWLERS # ============================================================ User-agent: AI2Bot Allow: / User-agent: Ai2Bot-Dolma Allow: / User-agent: Bytespider Allow: / User-agent: Diffbot Allow: / User-agent: ImagesiftBot Allow: / User-agent: Kangaroo Bot Allow: / User-agent: omgili Allow: / User-agent: omgilibot Allow: / User-agent: PetalBot Allow: / User-agent: Timpibot Allow: / User-agent: VelenPublicWebCrawler Allow: / User-agent: Webzio-Extended Allow: / User-agent: YouBot Allow: / # ============================================================ # SEO / RESEARCH CRAWLERS # ============================================================ User-agent: AhrefsBot Allow: / User-agent: SemrushBot Allow: / User-agent: MJ12bot Allow: / User-agent: DotBot Allow: / User-agent: Screaming Frog SEO Spider Allow: / User-agent: rogerbot Allow: / User-agent: SiteAuditBot Allow: / # ============================================================ # FEED READERS & AGGREGATORS # ============================================================ User-agent: feedly Allow: / User-agent: Feedspot Allow: / User-agent: Flipboard Allow: / User-agent: NewsBlur Allow: / # ============================================================ # ACCESSIBILITY & PERFORMANCE TESTING # ============================================================ User-agent: Lighthouse Allow: / User-agent: PTST Allow: / User-agent: Chrome-Lighthouse Allow: / # ============================================================ # CRAWL DELAY # Be polite — but don't slow down major engines # ============================================================ # No crawl-delay set. All bots are welcome at full speed. # If you experience server load issues, uncomment below: # User-agent: * # Crawl-delay: 1 # ============================================================ # SITEMAP LOCATION # ============================================================ Sitemap: https://parnellwellness.com/sitemap.xml # ============================================================ # ADDITIONAL DISCOVERY FILES # ============================================================ # LLM-specific context file (see llms.txt proposal) # https://parnellwellness.com/llms.txt # # humans.txt # https://parnellwellness.com/humans.txt # ============================================================