CODEWITHSUNDEEP
.htaccessmod-rewriteipbandwidth
flapane
flapane

Reputation: 543

IP banning via .htaccess not working

Liteserver on a Shared access host here.

I'm trying to get rid of a lot of bots who waste my resources for nothing.

I can successfully restrict access to some of them using a specific User Agent, but I can't ban their IP addresses, together with those of a lot of chinese ones who are constantly scanning my website. I am still seeing AhrefsBot IP (5.10.83.44) in the access log, even if its IP is banned (see the last line of the htaccess file). The rules should already be inherited by /gallery subfolder.

5.10.83.44 - - [07/Sep/2013:00:56:42 +0200] "GET /gallery/addfav.php?pid=858&referer=displayimage.php%3Fpid%3D858 HTTP/1.1" 302 156 "-" "Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)"

Root's .htaccess 

AddDefaultCharset UTF-8

<IfModule mod_headers.c>
  <FilesMatch "\.(js|css)$">
    Header append Vary Accept-Encoding
    Cache-Control: Private
  </FilesMatch>
</IfModule>

RewriteEngine on

#inherit from root htaccess and append at last, necessary in root too
RewriteOptions inherit

#block bad bots
RewriteCond %{HTTP_USER_AGENT} ^$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^spider$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^robot$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^crawl$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^(.*)AhrefsBot(.*) [OR]
RewriteCond %{HTTP_USER_AGENT} ^discovery$ [OR]
RewriteCond %{HTTP_USER_AGENT} ^Wget [OR]
RewriteCond %{HTTP_USER_AGENT} 360Spider [OR]
RewriteCond %{HTTP_USER_AGENT} Ezooms/1.0 [OR]
RewriteCond %{HTTP_USER_AGENT} MJ12bot/v1.4.4
RewriteRule ^(.*)$ http://go.away/

#include caching for images
<IfModule mod_expires.c>
    ExpiresActive On
    ExpiresByType image/gif "access plus 1 week"
    ExpiresByType image/jpg "access plus 1 week"
    ExpiresByType image/png "access plus 1 week"
    ExpiresByType image/x-icon "access plus 360 days"
    ExpiresByType text/css "access plus 1 day"
    ExpiresByType text/html "access plus 1 week"
    ExpiresByType text/javascript "access plus 1 week"  
    ExpiresByType text/x-javascript "access plus 1 week"
    ExpiresByType application/javascript "access plus 1 week"
    ExpiresByType application/x-javascript "access plus 1 week"
    ExpiresByType application/x-shockwave-flash "access plus 1 week"
    ExpiresByType font/truetype "access plus 1 month"
    ExpiresByType font/opentype "access plus 1 month"
    ExpiresByType application/x-font-otf "access plus 1 month"
</IfModule>


RewriteCond %{HTTP_HOST} ^id.foo.com$ [OR]
RewriteCond %{HTTP_HOST} ^www.id.foo.com$
RewriteRule ^/?$ "http\:\/\/foo\.myopenid\.com\/" [R=301,L]

Options +FollowSymLinks
RewriteCond %{HTTP_HOST} !^www\.
RewriteRule ^(.*)$ http://www.%{HTTP_HOST}/$1 [R=301,L]

#ban bots and stuff
order allow,deny
deny from 113.212.68.114
deny from 77.232.159.95
deny from 1.12.0.0/14
deny from 1.24.0.0/13
deny from 1.32.0.0/16
[...]
deny from 5.10.83.0/23
allow from all

/gallery subfolder .htaccess

RewriteEngine On

#inherit from root htaccess and append at last
RewriteOptions inherit

##/**************************************************
##  Coppermine 1.5.x Plugin - sef_urls
##  *************************************************
##  Copyright (c) 2003-2007 Coppermine Dev Team
##  *************************************************
##  This program is free software; you can redistribute it and/or modify
##  it under the terms of the GNU General Public License as published by
##  the Free Software Foundation; either version 3 of the License, or
##  (at your option) any later version.
##  ********************************************
##  $HeadURL$
##  $Revision$
##  $LastChangedBy$
##  $Date$
##  **************************************************/

#Options FollowSymLinks SymLinksIfOwnerMatch

# RewriteEngine on

#  Uncomment the following line if your webserver's
#  URL is not directly related to physical file paths.
#  Update "YOUR_COPPERMINE_ROOT" (just / for root)

#RewriteBase /gallery

#
# Language translation
#

RewriteRule cerca.html(.*) search.php$1 [NC]

RewriteRule contatti.html(.*) contact.php$1 [NC]

RewriteRule (.*)migliore(.*) $1toprated$2 [NC]

RewriteRule (.*)popolari(.*) $1topn$2 [NC]

RewriteRule (.*)osservazionedi(.*) $1lastcomby$2 [NC]

RewriteRule (.*)osservazione(.*) $1lastcom$2 [NC]

RewriteRule (.*)pagina(.*) $1page$2 [NC]

RewriteRule (.*)listautenti(.*) $1usermgr$2 [NC]

RewriteRule (.*)profilo(.*) $1profile$2 [NC]

RewriteRule (.*)miniature(.*) $1thumbnails$2 [NC]

RewriteRule (.*)mostra(.*) $1displayimage$2 [NC]

RewriteRule (.*)novitadi(.*) $1lastupby$2 [NC]

RewriteRule (.*)novita(.*) $1lastup$2 [NC]

RewriteRule (.*)cerca(.*) $1search$2 [NC]

RewriteRule (.*)top(.*) $1top_display_media$2 [NC]


#
# Rewrite usrmgr urls
#

RewriteRule usermgr-page-([0-9]*).html(.*) usermgr.php?page=$1$2 [NC]

RewriteRule usermgr.html(.*) usermgr.php$1 [NC]

#
# Rewrite index urls
#

RewriteRule index.html(.*) index.php$1 [NC]

RewriteRule index-([0-9]*)\.html(.*) index.php?cat=$1$2 [NC]

RewriteRule index-([0-9]*)-page-([0-9]*)\.html(.*) index.php?cat=$1&page=$2$3 [NC]


#
# Rewrite thumbnail urls
#

RewriteRule thumbnails-lastupby-([0-9]+)\.html(.*) thumbnails.php?album=lastupby&uid=$1$2 [NC]

RewriteRule thumbnails-lastcomby-([0-9]+)\.html(.*) thumbnails.php?album=lastcomby&uid=$1$2 [NC]

RewriteRule thumbnails-lastupby-([0-9]+)-([0-9]+)-page-([0-9]+)\.html(.*) thumbnails.php?album=lastupby&cat=$1&uid=$2&page=$3$4 [NC]

RewriteRule thumbnails-lastcomby-([0-9]+)-([0-9]+)-page-([0-9]+)\.html(.*) thumbnails.php?album=lastcomby&cat=$1&uid=$2&page=$3$4 [NC]

RewriteRule thumbnails-([a-z0-9]*)-([\-]?[0-9]*)\.html(.*) thumbnails.php?album=$1&cat=$2$3 [NC]

RewriteRule thumbnails-page-([0-9]*)-([a-z0-9]*).*\.html(.*) thumbnails.php?album=$2&page=$1$3 [NC]

RewriteRule thumbnails-([a-z0-9]*)-([\-]?[0-9]*)-page-([0-9]*)\.html(.*) thumbnails.php?album=$1&cat=$2&page=$3$4 [NC]

RewriteRule thumbnails-search-keyword-(.*)\.html(.*) thumbnails.php?album=search&keywords=on&search=$1$2 [NC]

RewriteRule thumbnails-search-(.*)\.html(.*) thumbnails.php?album=search&search=$1$2 [NC]

RewriteRule thumbnails-([0-9a-z]*).*\.html(.*) thumbnails.php?album=$1$2 [NC]


#
# Rewrite displayimage urls
#

RewriteRule displayimage-lastcom-([\-]?[0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)-page-([\-]?[0-9]+).html(.*) displayimage.php?album=lastcom&cat=$1&pid=$2&msg_id=$3&page=$4$5 [NC]

RewriteRule displayimage-([a-z0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)\.html(.*) displayimage.php?album=$1&cat=$2&pid=$3&uid=$4&msg_id=$5&page=$6$7 [NC]

RewriteRule displayimage-([a-z0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+)\.html(.*) displayimage.php?album=$1&cat=$2&pid=$3&uid=$4$5 [NC]

RewriteRule displayimage-([a-z0-9]+)-([\-]?[0-9]+)-([\-]?[0-9]+).*\.html(.*) displayimage.php?album=$1&cat=$2&pid=$3$4 [NC]

RewriteRule displayimage-([a-z0-9]+)-([\-]?[0-9]+).*\.html(.*) displayimage.php?album=$1&pid=$2$3 [NC]

RewriteRule displayimage-([0-9]+).*\.html(.*) displayimage.php?pid=$1$2 [NC]


#
# Rewrite users profiles
#

RewriteRule profile-op-([a-z0-9_]+)\.html(.*) profile.php?op=$1 [NC]

RewriteRule profile-([0-9]+).*\.html(.*) profile.php?uid=$1$2 [NC]

I'm sure I'm missing something, but unfortunately I have no clue.

Any hints?

Upvotes: 3

Views: 1510

Answers (1)

Jon Lin
Jon Lin

Reputation: 143946

According to that AhrefBot's link, this is all you need to do to stop that particular bot:

user-agent: AhrefsBot
disallow: /

By adding the above to a robots.txt file in your document root.

The other thing is this:

order allow,deny

Is in the wrong order. According to apache's mod_access documentation:

Allow,Deny

First, all Allow directives are evaluated; at least one must match, or the request is rejected. Next, all Deny directives are evaluated. If any matches, the request is rejected. Last, any requests which do not match an Allow or a Deny directive are denied by default.

Because you have "Allow from all", it gets evaluated first and the different "Deny"'s never get evaluated. So you want:

Order Deny,Allow

Upvotes: 1

Related Questions