Robots.txt
User-agent: *
Disallow: /ajax/
Disallow: /adminpanel/
Disallow: /myaccount/
Disallow: /adprint/
Disallow: /ad/leaflet/
Disallow: /ad/contact/
Disallow: /payment/
Disallow: /adding/confirm/
Disallow: /adding/confirmpage/
Disallow: /i2/ad/abuse/
Disallow: /m/ad/abuse/
Disallow: /apple-app-site-association
Disallow: /profile/
Disallow: /en/profile/
Disallow: *ref%*
Disallow: *search%*
Disallow: *search[*
Disallow: *sorting=*
Disallow: *minPhotoCount=*
Disallow: *product=*
Disallow: */changelang/*
Disallow: *back=*
Disallow: *invite=*
Disallow: /account
Disallow: /en/account
Disallow: /captchaChallenge
Disallow: /en/captchaChallenge
Disallow: /static-assets/hosted-resources-app/
Allow: /
# from olx.com.pk
User-agent: *
Disallow: /api/
Disallow: /post/
Disallow: /edit/
Disallow: /chat/
Disallow: /profile/
Disallow: /payments/
Disallow: /nf/
Allow: */api/categoryFields*
Allow: */api/featured-ads*
Allow: */api/locations*
# explorer legacy
Disallow: /deal_of_the_week/
Disallow: /en/deal_of_the_week/
Disallow: /featuredAgencies
Disallow: /en/featuredAgencies
Disallow: /recommended_properties*
Disallow: /profolio/
Disallow: /en/profolio/
Disallow: /cp/
Disallow: /en/cp/
Disallow: /nova/
Disallow: /en/nova/
# disallow all urls with parameters
Disallow: *filter=*
# disallow crawl on urls with model in path and free search
Disallow: */model-*/q-*
# contains urls which aren't of use for SEO
Disallow: /motors*?
Disallow: /en/motors*?
# path on search urls used to retrieve filters as json
Disallow: */raw-filters*
(Nice to have)