User-agent: Scrapy
Allow: /

User-agent: *
Disallow: /cgi-bin
Disallow: /wp-
Disallow: /wp/
Disallow: /?
Disallow: */comments
Disallow: /users/
Disallow: */trackback
Disallow: */feed
Disallow: */rss
Disallow: */embed*
Allow: /wp-admin/admin-ajax.php
Allow: /wp-includes/css/dist/
Allow: /wp-content/cache/min/1/
Allow: /wp-content/plugins/wp-rocket/
Allow: /wp-content/themes/closers/fonts/
Allow: /wp-content/uploads/
Allow: */
Disallow: /terms-conditions
Disallow: /privacy-policy
Disallow: /thank-you
Disallow: /tag/
Disallow: /category/
Disallow: /resource-type/
Disallow: /testimonials-category/
Disallow: /wp-contet/uploads/*/*/*.pdf
Disallow: /wp-contet/uploads/*/*/*.doc
Disallow: /wp-contet/uploads/*/*/*.docx
Disallow: /*.pdf
Disallow: /*.doc
Disallow: /*.docx
Disallow: /*gclid=*
Disallow: /*fbclid=*
Disallow: /*?utm_source=
Disallow: /*utm_medium=
Disallow: /*utm_campaign=
Disallow: /*utm_content=
Disallow: /*utm_term=
Allow: */favicon/*.png

Sitemap: https://www.websiteclosers.com/sitemap_index.xml