# robots.txt for https://www.ricardo.ch/

User-agent: *

# English pages until release
Disallow: /en/
Disallow: /en$

# Don't try to index the login page, you will get a Cloudflare challenge
Disallow: /de/login
Disallow: /fr/login
Disallow: /it/login
Disallow: /de/regist
Disallow: /fr/regist
Disallow: /it/regist

# After login pages
Disallow: /*/bulk/landing
Disallow: /*/my-ricardo
Disallow: /*/bookkeeping
Disallow: /de/profile
Disallow: /fr/profile
Disallow: /it/profile
Disallow: /de/list
Disallow: /fr/list
Disallow: /it/list

# SRP
Disallow: /*/s/

# CSRP
Disallow:/*/c/*/*/
Disallow: /*/c/*?*,*
Disallow: /en/c/*?
Allow: /de/c/o/*
Allow: /*/c/*?

# Sellers
Disallow: /*/shop/*/offers/*/
Disallow: /*/shop/*/offers*?page=*&
Allow: /*/shop/*/offers?page=
Allow: /*/shop/*/offers/?page=
Disallow: /*/shop/*/ratings*?

# Other params
Disallow: /*?*ar=
Disallow: /*?*qcn=
Disallow: /*?*nextOffset
Disallow: /*?*next_offset
Disallow: /*?*category=

# Supporting files
Disallow: /api/browser-statistics/
Disallow: /api/frontend/search-autocomplete
Disallow: /api/frontend/notifications
Disallow: /api/mfa/categories
Allow: /api/mfa/categories/*/promo-offers
Disallow: /api/mfa/notifications
Disallow: /api/mfa/user/save-seller-id
Disallow: /px/pv
Disallow: /api/listing-form
Disallow: /marketplace-spa/api/questions
Disallow: /marketplace-spa/api/fee
Disallow: /marketplace-spa/api/pdp
Allow: /marketplace-spa/api/pdp/youtube

# Legacy
Disallow: /dataservice/
Disallow: /pages/
Disallow: /ajax/
Disallow: /viewitem.aspx
Disallow: /*feed.xml
Disallow: /online-shop/
Disallow: /shop/
Disallow: /ratings/
Disallow: /pages/*/fr.php
Disallow: /auk/
Disallow: /acheter/
Disallow: /kaufen/
Disallow: /zubehoer/
Disallow: */v/
Disallow: */w/

# Disallow cloudflare /cdn-cgi/ endpoint.
# See https://developers.cloudflare.com/fundamentals/get-started/reference/cdn-cgi-endpoint/
Disallow: /cdn-cgi/

# Disallow commercial bots we don't like
User-agent: psbot
User-agent: Yandex
User-agent: PetalBot
User-agent: Mail.RU_Bot
User-agent: MegaIndex
User-agent: Baiduspider
User-agent: 360Spider
User-agent: Yisouspider
User-agent: Bytespider
User-agent: Sogou web spider
User-agent: Sogou inst spider
User-agent: proximic
User-agent: ADmantX
User-agent: Ahrefs
User-agent: OnCrawl
User-agent: Seekport Crawler
User-agent: SEMrushBot
User-agent: BLEXBot
User-agent: MJ12bot
User-agent: ZoomBot
User-agent: Linkbot
User-agent: GPTBot
User-agent: dotbot
Disallow: /

# User agent names for Google AdsBot can be found here: https://support.google.com/webmasters/answer/1061943?hl=en
User-agent: AdsBot-Google-Mobile
User-agent: AdsBot-Google
Disallow: /en/
Disallow: /en$
Disallow: /de/login
Disallow: /fr/login
Disallow: /it/login
Disallow: /de/regist
Disallow: /fr/regist
Disallow: /it/regist
Disallow: /*/bulk/landing
Disallow: /*/my-ricardo
Disallow: /*/bookkeeping
Disallow: /de/profile
Disallow: /fr/profile
Disallow: /it/profile
Disallow: /de/list
Disallow: /fr/list
Disallow: /it/list
Disallow: /*/s$
Disallow: /*/s/$
Disallow: /*/s/*?
Disallow: /*/shop/*/
Disallow: /*?*sort=
Disallow: /*?*ar=
Disallow: /*?*qcn=
Disallow: /*?*ignore_dominant=
Disallow: /*?*nextOffset
Disallow: /*?*next_offset
Disallow: /*?*category=
Disallow: /cdn-cgi/
Disallow: /api/browser-statistics/
Disallow: /api/frontend/search-autocomplete
Disallow: /api/frontend/notifications
Disallow: /api/frontend/categories
Allow: /api/mfa/categories/*/promo-offers
Disallow: /px/pv
Disallow: /marketplace-spa/api/
Disallow: /api/mfa/
Disallow: /assets/search/
Disallow: /api/listing-form

# Limit criteo crawler to 5 req/sec max as it generates imaginary-wrapper traffic peaks 
User-agent: CriteoBot/0.1  
Crawl-delay: 0.2

# Allow Oracle Data Cloud Crawler
User-agent: grapeshot
Disallow: