Index: nselib/httpspider.lua
===================================================================
--- nselib/httpspider.lua (revision 31056)
+++ nselib/httpspider.lua (working copy)
@@ -57,6 +57,22 @@
-- HEAD instead of GET for files that do not have extensions indicating
-- that they are webpages (the list of webpage extensions is located in
-- nselib/data/http-web-files-extensions.lst)
+-- @args httpspider.blacklistforscraping the crawler won't scrape the
+-- resources defined in this table. The table should contain
+-- dictionaries with 'extension' and 'location' keys defined. For
+-- example, if i set this argument to { { extension = "*", location =
+-- "outsidehost"}, { extension = "js", location = "withinoroutsidehost"}
+-- }, the crawler won't scrape any resources outside the target or any
+-- JS files within or outside the target host. The 'extension' key
+-- may hold valid strings as extensions or the "*" symbol for all
+-- extensions (hence, all URLs). The 'location' may contain three values:
+-- 'withinhost' for URLs that point inside the host target, 'outsidehost'
+-- for URLs that point outside the host target and 'withinoroutsidehost'
+-- for URLs that point anywhere. Please, note that by using this argument,
+-- the crawler will spider the defined URLs (but it won't scrape them). If
+-- you need to define a blacklist for spidering, please check the
+-- 'withinhost', 'withindomain', 'noblacklist' and 'maxdepth' arguments.
+-- (default: {})
--
local coroutine = require "coroutine"
@@ -85,42 +101,6 @@
o.timeout = options.timeout or 10000
o.whitelist = o.whitelist or {}
o.blacklist = o.blacklist or {}
- local removewww = function(url) return string.gsub(url, "^www%.", "") end
-
- if ( o.withinhost == true or o.withindomain == true ) then
- -- set up the appropriate matching functions
- if ( o.withinhost ) then
- o.withinhost = function(u)
- local parsed_u = url.parse(tostring(u))
-
- if ( o.base_url:getPort() ~= 80 and o.base_url:getPort() ~= 443 ) then
- if ( tonumber(parsed_u.port) ~= tonumber(o.base_url:getPort()) ) then
- return false
- end
- elseif ( parsed_u.scheme ~= o.base_url:getProto() ) then
- return false
- -- if urls don't match only on the "www" prefix, then they are probably the same
- elseif ( parsed_u.host == nil or removewww(parsed_u.host:lower()) ~= removewww(o.base_url:getHost():lower()) ) then
- return false
- end
- return true
- end
- else
- o.withindomain = function(u)
- local parsed_u = url.parse(tostring(u))
- if ( o.base_url:getPort() ~= 80 and o.base_url:getPort() ~= 443 ) then
- if ( tonumber(parsed_u.port) ~= tonumber(o.base_url:getPort()) ) then
- return false
- end
- elseif ( parsed_u.scheme ~= o.base_url:getProto() ) then
- return false
- elseif ( parsed_u.host == nil or parsed_u.host:sub(-#o.base_url:getDomain()):lower() ~= o.base_url:getDomain():lower() ) then
- return false
- end
- return true
- end
- end
- end
setmetatable(o, self)
self.__index = self
return o
@@ -153,6 +133,41 @@
return o
end,
+ removewww = function(url) return string.gsub(url, "^www%.", "") end,
+
+ iswithinhost = function(self, u, o)
+
+ local parsed_u = url.parse(tostring(u))
+
+ if ( o.base_url:getPort() ~= 80 and o:getPort() ~= 443 ) then
+ if ( tonumber(parsed_u.port) ~= tonumber(o.base_url:getPort()) ) then
+ return false
+ end
+ elseif ( parsed_u.scheme ~= o.base_url:getProto() ) then
+ return false
+ -- if urls don't match only on the "www" prefix, then they are probably the same
+ elseif ( parsed_u.host == nil or self.removewww(parsed_u.host:lower()) ~= self.removewww(o.base_url:getHost():lower()) ) then
+ return false
+ end
+ return true
+ end,
+
+ iswithindomain = function(self, u, o)
+
+ local parsed_u = url.parse(tostring(u))
+
+ if ( o.base_url:getPort() ~= 80 and o.base_url:getPort() ~= 443 ) then
+ if ( tonumber(parsed_u.port) ~= tonumber(o.base_url:getPort()) ) then
+ return false
+ end
+ elseif ( parsed_u.scheme ~= o.base_url:getProto() ) then
+ return false
+ elseif ( parsed_u.host == nil or parsed_u.host:sub(-#o.base_url:getDomain()):lower() ~= o.base_url:getDomain():lower() ) then
+ return false
+ end
+ return true
+ end,
+
-- is the link absolute or not?
isAbsolute = function(url)
-- at this point we don't care about the protocol
@@ -171,6 +186,11 @@
-- @return link string containing the absolute link
createAbsolute = function(base_url, rel_url, base_href)
+ -- is protocol-relative?
+ if rel_url:match("^//") then
+ return ("%s%s%s"):format(base_url:getProto(), ":", rel_url)
+ end
+
-- is relative with leading slash? ie /dir1/foo.html
local leading_slash = rel_url:match("^/")
rel_url = rel_url:match("^/?(.*)") or '/'
@@ -226,7 +246,7 @@
end
end
end,
-
+
validate_link = function(self, url)
local valid = true
@@ -248,7 +268,7 @@
-- withindomain trumps any whitelisting
if ( self.options.withindomain ) then
- if ( not(self.options.withindomain(url)) ) then
+ if ( not(self.iswithindomain(self, url, self.options)) ) then
stdnse.print_debug(2, "%s: Link is not within domain: %s", LIBRARY_NAME, tostring(url))
return false
end
@@ -256,7 +276,7 @@
-- withinhost trumps any whitelisting
if ( self.options.withinhost ) then
- if ( not(self.options.withinhost(url)) ) then
+ if ( not(self.iswithinhost(self, url, self.options)) ) then
stdnse.print_debug(2, "%s: Link is not within host: %s", LIBRARY_NAME, tostring(url))
return false
end
@@ -503,6 +523,7 @@
-- maxpagecount - the maximum amount of pages to retrieve
-- withinhost - stay within the host of the base_url
-- withindomain - stay within the base_url domain
+ -- blacklistforscraping - don't scrape resources defined in this table
-- scriptname - should be set to SCRIPT_NAME to enable
-- script specific arguments.
-- redirect_ok - redirect_ok closure to pass to http.get function
@@ -553,7 +574,7 @@
if ( not(o.options.maxpagecount) ) then
o.options.maxpagecount = tonumber(stdnse.get_script_args("httpspider.maxpagecount"))
end
-
+
if ( not(o.options.noblacklist) ) then
o:addDefaultBlacklist()
end
@@ -631,6 +652,7 @@
-- This way the script can alert the user of the details by calling
-- getError()
crawl_thread = function(self, response_queue)
+
local condvar = nmap.condvar(response_queue)
if ( false ~= self.options.withinhost and false ~= self.options.withindomain ) then
@@ -716,11 +738,23 @@
url.path = link
end
end
- -- if we have a response, proceed scraping it
- if ( response.body ) then
- local links = LinkExtractor:new(url, response.body, self.options):getLinks()
- self.urlqueue:add(links)
- end
+ local noscraping = false
+
+ for _, resource in ipairs(self.options.blacklistforscraping) do
+ if (string.sub(tostring(url), -string.len(resource.extension) - 1)== "." .. resource.extension or
+ resource.extension == "*") then
+ if (resource.location == "outsidehost" and not(LinkExtractor:iswithinhost(url, self.options))) or
+ (resource.location == "withinhost" and LinkExtractor:iswithinhost(url, self.options)) or
+ (resource.location == "withinoroutsidehost") then
+ noscraping = true
+ end
+ end
+ end
+ -- if we have a response and the resource is not in the blacklist for scraping, proceed scraping it
+ if response.body and noscraping == false then
+ local links = LinkExtractor:new(url, response.body, self.options):getLinks()
+ self.urlqueue:add(links)
+ end
else
response = { body = "", headers = {} }
end
@@ -763,6 +797,9 @@
if ( nil == self.options.useheadfornonwebfiles ) then
self.options.useheadfornonwebfiles = stdnse.get_script_args(sn .. ".useheadfornonwebfiles")
end
+ if ( nil == self.options.blacklistforscraping ) then
+ self.options.blacklistforscraping = stdnse.get_script_args(sn .. ".blacklistforscraping")
+ end
end,
-- Loads the argument on a library level
@@ -790,6 +827,9 @@
if ( nil == self.options.useheadfornonwebfiles ) then
self.options.useheadfornonwebfiles = stdnse.get_script_args(ln .. ".useheadfornonwebfiles")
end
+ if ( nil == self.options.blacklistforscraping ) then
+ self.options.blacklistforscraping = stdnse.get_script_args(ln .. ".blacklistforscraping")
+ end
end,
-- Loads any defaults for arguments that were not set
@@ -819,8 +859,11 @@
self.options.withinhost = tobool(self.options.withinhost)
self.options.withindomain = tobool(self.options.withindomain)
self.options.noblacklist = tobool(self.options.noblacklist)
- self.options.useheadfornonwebfiles = tobool(self.options.useheadfornonwebfiles)
+ self.options.useheadfornonwebfiles = tobool(self.options.useheadfornonwebfiles)
+ if type(self.options.blacklistforscraping) ~= "table" then
+ self.options.blacklistforscraping = {}
+ end
if ( self.options.withinhost == nil ) then
if ( self.options.withindomain ~= true ) then
self.options.withinhost = true
@@ -830,9 +873,12 @@
end
if ( self.options.withindomain == nil ) then
self.options.withindomain = false
- end
+ end
+ if ( self.options.blacklistforscraping == nil ) then
+ self.options.blacklistforscraping = {}
+ end
self.options.maxdepth = self.options.maxdepth or 3
- self.options.maxpagecount = self.options.maxpagecount or 20
+ self.options.maxpagecount = self.options.maxpagecount or 20
self.url = self.url or '/'
end,
@@ -863,6 +909,7 @@
-- does the crawling
crawl = function(self)
+
self.response_queue = self.response_queue or {}
local condvar = nmap.condvar(self.response_queue)
if ( not(self.thread) ) then