User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /*?
Disallow: /wp-login.php
Disallow: */trackback
Disallow: /*/feed
Disallow: /*/comments
Disallow: /cgi-bin
Disallow: /*.php$
Disallow: /*.inc$
Disallow: /*.gz
Disallow: /*.cgi
Disallow: /FICHIERS
Allow: /index.php
Allow: /*css?*
Allow: /*js?*
Allow: /*google*
Allow: /*facebook*
Allow: /*twitter*
Allow: /*mail*
Allow: /*?*ajax*
allow: /wp-content/plugins/*font*