在文件目录/www/server/nginx/conf文件夹下面，新建一个文件，命名为:agent_deny.conf
#禁止Scrapy等工具的抓取
if ($http_user_agent ~* (Scrapy|Curl|HttpClient)) {
return 403;
}
#禁止指定UA及UA为空的访问
if ($http_user_agent ~* "FeedDemon|Indy Library|Alexa Toolbar|AskTbFXTV|AhrefsBot|CrawlDaddy|CoolpadWebkit|Java|Feedly|UniversalFeedParser|ApacheBench|Microsoft URL Control|Swiftbot|ZmEu|oBot|jaunty|Python-urllib|lightDeckReports Bot|YYSpider|DigExt|webmeup-crawler|dataforseo|grapeshot|HttpClient|MJ12bot|heritrix|Bytespider|Ezooms|JikeSpider|SemrushBot|ahrefs|semrush|comscore|seo|opensiteexplorer|semrush|^$" ) {
return 403;
}
#禁止非GET|HEAD|POST方式的抓取
if ($request_method !~ ^(GET|HEAD|POST)$) {
return 403;
}
找到网站设置里面:配置文件 的第7行左右，写入代码：include agent_deny.conf;
