User-agent: Storebot
Disallow:
Crawl-delay: 30

User-agent: meta-externalagent
Disallow:
Crawl-delay: 30

User-agent: bingbot
Disallow:
Crawl-delay: 30

User-agent: *
Disallow: /lib/
Disallow: /*.php$
Disallow: /pkginfo/
Disallow: /report/
Disallow: /var/
Disallow: /customer/
Disallow: /sendfriend/
Disallow: /review/
Disallow: /*SID=
Disallow: /*?
 
# Disable onestepcheckout, customer account, checkout and search
Disallow: /onestepcheckout/
Disallow: /sales/order/
Disallow: /wishlist/
Disallow: /checkout/ 
Disallow: /catalogsearch/
Disallow: /searchautocomplete/
 
# Disable common folders
Disallow: /app/
Disallow: /bin/
Disallow: /dev/
Disallow: /lib/
Disallow: /phpserver/
 
# Disable Tag & Review (Avoid duplicate content)
Disallow: /tag/
Disallow: /review/
 
# Common files
Disallow: /composer.json
Disallow: /composer.lock
Disallow: /CONTRIBUTING.md
Disallow: /CONTRIBUTOR_LICENSE_AGREEMENT.html
Disallow: /COPYING.txt
Disallow: /Gruntfile.js
Disallow: /LICENSE.txt
Disallow: /LICENSE_AFL.txt
Disallow: /nginx.conf.sample
Disallow: /package.json
Disallow: /php.ini.sample
Disallow: /RELEASE_NOTES.txt
 
# Disable sorting (Avoid duplicate content)
Disallow: /*?*product_list_mode=
Disallow: /*?*product_list_order=
Disallow: /*?*product_list_limit=
Disallow: /*?*product_list_dir=
 
# Disable version control folders and others
Disallow: /*.git
Disallow: /*.CVS
Disallow: /*.Zip$
Disallow: /*.Svn$
Disallow: /*.Idea$
Disallow: /*.Sql$
Disallow: /*.Tgz$
 
# Disable custom folders
Disallow: /bridge2cart/
Disallow: /custom/
Disallow: /db/
Disallow: /extensions__12-11-2018/
Disallow: /FilterScript/
Disallow: /generated/
Disallow: /generated--/
Disallow: /generated.bk/
Disallow: /magento/
Disallow: /magewrox_zips/
Disallow: /magmi/
Disallow: /reloadseo/
Disallow: /setmeta/
Disallow: /setup/
Disallow: /update/
Disallow: /vendor/
Disallow: /warmer/
Disallow: /files/
Disallow: /pub/media/duplicate_products/
Disallow: /pub/media/product_files/
Disallow: /pub/media/synccsv/
Disallow: /pub/media/indexer_products/
Disallow: /cpos.txt
Disallow: /delete_user.txt
Disallow: /events.txt
Disallow: /file.txt
Disallow: /get_url.txt
Disallow: /get_url1.txt
Disallow: /pub/media/debug.txt
Disallow: /pub/media/import/cache_debug.txt
 
Disallow: /*.log
Disallow: /*.sh

Sitemap: https://www.pacoperfumerias.co.uk/sitemap.xml

Allow: /*?p=
Disallow: /catalog/category/view/
Sitemap: https://www.pacoperfumerias.co.uk/sitemap.xml