# ===========================================
# Robots.txt for faremanino.com
# Fare Manino - Location maison Huahine
# Villa bord de mer Polynésie Française
# Last Updated: 2026-01-25
# ===========================================

# ========== MAJOR SEARCH ENGINES ==========

# Google (web, images, video, news, shopping)
User-agent: Googlebot
Allow: /
Crawl-delay: 1

User-agent: Googlebot-Image
Allow: /images/
Allow: /assets/
Allow: /

User-agent: Googlebot-Video
Allow: /

User-agent: Googlebot-News
Allow: /blog/
Allow: /en/blog/
Allow: /

User-agent: Storebot-Google
Allow: /

User-agent: Google-InspectionTool
Allow: /

User-agent: GoogleOther
Allow: /

# Bing & Microsoft
User-agent: Bingbot
Allow: /
Crawl-delay: 1

User-agent: msnbot
Allow: /

User-agent: BingPreview
Allow: /

User-agent: adidxbot
Allow: /

# Yahoo
User-agent: Slurp
Allow: /
Crawl-delay: 1

# DuckDuckGo
User-agent: DuckDuckBot
Allow: /

# Yandex (Russia)
User-agent: Yandex
Allow: /
Crawl-delay: 2

User-agent: YandexBot
Allow: /

User-agent: YandexImages
Allow: /images/
Allow: /assets/
Allow: /

User-agent: YandexMobileBot
Allow: /

# Baidu (China)
User-agent: Baiduspider
Allow: /
Crawl-delay: 2

User-agent: Baiduspider-image
Allow: /images/
Allow: /assets/
Allow: /

User-agent: Baiduspider-video
Allow: /

# Sogou (China)
User-agent: Sogou
Allow: /

User-agent: Sogou web spider
Allow: /

# Naver (Korea)
User-agent: Yeti
Allow: /

# Qwant (France/Europe)
User-agent: Qwantify
Allow: /

# Ecosia
User-agent: Ecosia
Allow: /

# Seznam (Czech Republic)
User-agent: SeznamBot
Allow: /

# Brave Search
User-agent: Brave
Allow: /

# Mojeek
User-agent: MojeekBot
Allow: /

# ========== AI SEARCH & ASSISTANTS ==========

# OpenAI GPT / ChatGPT
User-agent: GPTBot
Allow: /

User-agent: ChatGPT-User
Allow: /

User-agent: OAI-SearchBot
Allow: /

# Google AI / Gemini / Bard
User-agent: Google-Extended
Allow: /

# Anthropic Claude
User-agent: anthropic-ai
Allow: /

User-agent: Claude-Web
Allow: /

User-agent: ClaudeBot
Allow: /

# Meta AI
User-agent: FacebookBot
Allow: /

User-agent: Meta-ExternalAgent
Allow: /

User-agent: Meta-ExternalFetcher
Allow: /

# Microsoft Copilot
User-agent: CopilotBot
Allow: /

# Perplexity AI
User-agent: PerplexityBot
Allow: /

# You.com AI
User-agent: YouBot
Allow: /

# Cohere AI
User-agent: cohere-ai
Allow: /

# Apple AI / Siri
User-agent: Applebot
Allow: /

User-agent: Applebot-Extended
Allow: /

# Amazon Alexa
User-agent: Amazonbot
Allow: /

# Mistral AI
User-agent: MistralBot
Allow: /

# ========== SOCIAL MEDIA CRAWLERS ==========

# Facebook / Instagram
User-agent: facebookexternalhit
Allow: /

User-agent: Facebot
Allow: /

# Twitter/X
User-agent: Twitterbot
Allow: /

# LinkedIn
User-agent: LinkedInBot
Allow: /

# Pinterest
User-agent: Pinterest
Allow: /

User-agent: Pinterestbot
Allow: /

# WhatsApp
User-agent: WhatsApp
Allow: /

# Telegram
User-agent: TelegramBot
Allow: /

# Slack
User-agent: Slackbot
Allow: /

User-agent: Slackbot-LinkExpanding
Allow: /

# Discord
User-agent: Discordbot
Allow: /

# Snapchat
User-agent: Snapchat
Allow: /

# TikTok
User-agent: TikTokBot
Allow: /

User-agent: Bytespider
Allow: /

# Reddit
User-agent: redditbot
Allow: /

# ========== TRAVEL & BOOKING CRAWLERS ==========

# TripAdvisor
User-agent: TripAdvisorBot
Allow: /

# Airbnb
User-agent: Airbnb
Allow: /

# Booking.com
User-agent: Bookingcom
Allow: /

# Expedia
User-agent: Expediabot
Allow: /

# Kayak
User-agent: Kayak
Allow: /

# Trivago
User-agent: Trivago
Allow: /

# ========== SEO & ANALYTICS TOOLS ==========

# Semrush
User-agent: SemrushBot
Allow: /

# Ahrefs
User-agent: AhrefsBot
Allow: /
Crawl-delay: 5

# Moz
User-agent: rogerbot
Allow: /
Crawl-delay: 5

User-agent: DotBot
Allow: /

# Majestic
User-agent: MJ12bot
Allow: /
Crawl-delay: 5

# Screaming Frog
User-agent: Screaming Frog SEO Spider
Allow: /

# Sistrix
User-agent: Sistrix
Allow: /

# Ubersuggest
User-agent: Ubersuggest
Allow: /

# ========== ARCHIVE & RESEARCH ==========

# Internet Archive
User-agent: ia_archiver
Allow: /

# Common Crawl
User-agent: CCBot
Allow: /

# Wayback Machine
User-agent: archive.org_bot
Allow: /

# ========== PRICE COMPARISON & AGGREGATORS ==========

User-agent: PriceSpider
Allow: /

User-agent: Pricewatch
Allow: /

# ========== DEFAULT RULE ==========

User-agent: *
Allow: /

# ========== BLOCKED PATHS ==========
# (Applied to all user agents)

Disallow: /settings
Disallow: /api/
Disallow: /*.json$
Disallow: /reservation/succes
Disallow: /en/booking/success
Disallow: /_
Disallow: /supabase/
Disallow: /settings
Disallow: /*/settings

# ========== SITEMAP LOCATIONS ==========

# Main sitemap index (recommended)
Sitemap: https://faremanino.com/sitemap-index.xml

# Individual sitemaps
Sitemap: https://faremanino.com/sitemap.xml
Sitemap: https://faremanino.com/sitemap-images.xml
Sitemap: https://faremanino.com/sitemap-pwa.xml

# ========== CRAWL GUIDELINES ==========
# This site is optimized for vacation rental search
# Content includes: accommodation, French Polynesia travel, Huahine tourism
# Languages supported: French (primary), English
# Updates: Weekly for main pages, monthly for blog articles
# Key pages: /location-huahine, /villa-huahine, /hebergement-huahine (FR)
#            /en/huahine-rental, /en/huahine-villa, /en/huahine-accommodation (EN)
# PWA: Progressive Web App with offline support and push notifications