Nmap Development mailing list archives
Re: [NSE] WHOIS - Now with Queuing and Cached Results.
From: jah <jah () zadkiel plus com>
Date: Wed, 23 Apr 2008 03:51:03 +0100
On 09/04/2008 02:32, majek04 wrote:
On Fri, Feb 15, 2008 at 3:54 AM, jah <jah () zadkiel plus com> wrote:My problem is; how to make a coroutine wait?In os.nse [1] I use this solution, based on pcap (raw sockets): -- hack used only to go back to event loop local pcap = nmap.new_socket() function pcap_callback() return 'xxx' end pcap:pcap_open(interface, 64, 0, pcap_callback, "tcp[0] == 0xDE and ((tcp[tcpflags] & tcp-fin) != 0) " ) -- go back to the event loop for a while pcap:set_timeout(XXX) -- XXXms pcap:pcap_register('aaa') _, _, _, _, _ = pcap:pcap_receive() But we really should create something more standard, like nmap.sleep(). Maybe it's a good idea for SoC?
nmap.sleep() sounds like a good idea to me. Until then, I've used Marek's pcap hack for whois.nse to enforce a strict queue for each whois service. This enables caching of results and means that a) for a given net-range only one query per whois service is required for any targets within that range and b) a given whois service will have only one query outstanding at any time - queries to different whois services can still happen concurrently. Once a record has been found for a target, any further targets within that range will point to the same result, rather than print it again. I've attached the new and improved whois.nse and welcome any feedback - especially if you manage to break it. Cheers, jah
id = "WHOIS"
description = [[
Attempts to retrieve WHOIS information about the range of IP addresses
that include the target IP address. If a record contains more than
one range of addresses, information about the smallest range should be
returned. With output verbosity, information about Persons or Roles
associated with the range will be returned.
]]
--[[
USAGE
nmap <target> --script whois
nmap <target> --script discovery
The default number and query order of Whois services may be overridden
by supplying the script argument "whodb" thus:
nmap <target> --script whois --script-args whodb=ripe
nmap <target> --script whois --script-args whodb=arin+ripe+apnic
nmap <target> --script whois --script-args whois={whodb=apnic*jpnic}
(N.B. commas or semi-colons should not be used to delimit arguments)
--]]
author = "jah <jah at zadkiel.plus.com>"
license = "Same as Nmap--See http://nmap.org/book/man-legal.html";
categories = {"discovery"}
runlevel = 1.0
require "ipOps"
require "stdnse"
-- HOSTRULE
-- true for any non-private IP address
hostrule = function(host, port)
return not ipOps.isPrivate(host.ip)
end
-- ORDERDEFN
-- define the number and order of whois databases to query
local orderdefn = {"arin","ripe","apnic"}
-- FIELDS
-- define the fields from which to gather values
local fields_meta = {
rpsl = {
ob_exist = "\n%s*[Ii]netnum:%s*.-\n",
ob_netnum = {ob_start = "\n%s*[Ii]netnum:%s*.-\n",
ob_end = "\n%s*[Ss]ource:%s*.-\n\n",
inetnum = "\n%s*[Ii]netnum:%s*(.-)\n",
netname = "\n%s*[Nn]et[\-]-[Nn]ame:%s*(.-)\n",
nettype = "\n%s*[Nn]et[\-]-[Tt]ype:%s*(.-)\n",
descr = "[Dd]escr:[^\n][%s]*(.-)\n",
country = "\n%s*[Cc]ountry:%s*(.-)\n",
status = "\n%s*[Ss]tatus:%s*(.-)\n",
source = "\n%s*[Ss]ource:%s*(.-)\n"},
ob_org = { ob_start = "\n%s*[Oo]rgani[sz]ation:%s*.-\n",
ob_end = "\n%s*[Ss]ource:%s*.-\n\n",
organisation = "\n%s*[Oo]rgani[sz]ation:%s*(.-)\n",
orgname = "\n%s*[Oo]rg[\-]-[Nn]ame:%s*(.-)\n",
descr = "[Dd]escr:[^\n][%s]*(.-)\n",
email = "\n%s*[Ee][\-]-[Mm]ail:%s*(.-)\n"},
ob_role = { ob_start = "\n%s*[Rr]ole:%s*.-\n",
ob_end = "\n%s*[Ss]ource:%s*.-\n\n",
role = "\n%s*[Rr]ole:%s*(.-)\n",
email = "\n%s*[Ee][\-]-[Mm]ail:%s*(.-)\n"},
ob_persn = { ob_start = "\n%s*[Pp]erson:%s*.-\n",
ob_end = "\n%s*[Ss]ource:%s*.-\n\n",
person = "\n%s*[Pp]erson:%s*(.-)\n",
email = "\n%s*[Ee][\-]-[Mm]ail:%s*(.-)\n"} },
arin = {
ob_exist = "\n%s*[Nn]et[\-]-[Rr]ange:.-\n",
ob_netnum = {ob_start = "\n%s*[Nn]et[\-]-[Rr]ange:.-\n",
ob_end = "\n\n",
netrange = "\n%s*[Nn]et[\-]-[Rr]ange:(.-)\n",
netname = "\n%s*[Nn]et[\-]-[Nn]ame:(.-)\n",
nettype = "\n%s*[Nn]et[\-]-[Tt]ype:(.-)\n"},
ob_org = {ob_start = "\n%s*[Oo]rg[\-]-[Nn]ame:.-\n",
ob_end = "\n\n",
orgname = "\n%s*[Oo]rg[\-]-[Nn]ame:(.-)\n",
orgid = "\n%s*[Oo]rg[\-]-[Ii][Dd]:(.-)\n",
stateprov = "\n%s*[Ss]tate[\-]-[Pp]rov:(.-)\n",
country = "\n%s*[Cc]ountry:(.-)\n"},
ob_cust = {ob_start = "\n%s*[Cc]ust[\-]-[Nn]ame:.-\n",
ob_end = "\n\n",
custname = "\n%s*[Cc]ust[\-]-[Nn]ame:(.-)\n",
stateprov = "\n%s*[Ss]tate[\-]-[Pp]rov:(.-)\n",
country = "\n%s*[Cc]ountry:(.-)\n"},
ob_persn = {ob_start = "\n%s*[Oo]rg[\-]-[Tt]ech[\-]-[Nn]ame:.-\n",
ob_end = "\n\n",
orgtechname =
"\n%s*[Oo]rg[\-]-[Tt]ech[\-]-[Nn]ame:(.-)\n",
orgtechemail =
"\n%s*[Oo]rg[\-]-[Tt]ech[\-]-[Ee][\-]-[Mm]ail:(.-)\n"} },
lacnic = {
ob_exist = "\n%s*[Ii]netnum:%s*.-\n",
ob_netnum = {ob_start = "\n%s*[Ii]netnum:%s*.-\n",
ob_end = "\n\n",
inetnum = "\n%s*[Ii]netnum:%s*(.-)\n",
owner = "\n%s*[Oo]wner:%s*(.-)\n",
ownerid = "\n%s*[Oo]wner[\-]-[Ii][Dd]:%s*(.-)\n",
responsible = "\n%s*[Rr]esponsible:%s*(.-)\n",
country = "\n%s*[Cc]ountry:%s*(.-)\n",
source = "\n%s*[Ss]ource:%s*(.-)\n"},
ob_persn = {ob_start = "\n%s*[Pp]erson:%s*.-\n",
ob_end = "\n\n",
person = "\n%s*[Pp]erson:%s*(.-)\n",
email = "\n%s*[Ee][\-]-[Mm]ail:%s*(.-)\n"} },
jpnic = {
ob_exist = "\n%s*[Nn]etwork%s-[Ii]nformation:%s*.-\n",
ob_netnum = {ob_start = "\[[Nn]etwork%s*[Nn]umber\]%s*.-\n",
ob_end = "\n\n",
inetnum = "\[[Nn]etwork%s*[Nn]umber\]%s*(.-)\n",
netname = "\[[Nn]etwork%s*[Nn]ame\]%s*(.-)\n",
orgname = "\[[Oo]rganization\]%s*(.-)\n"} }
}
-- WHOISDB
-- there can be more dbs defined here than in orderdefn, those extra will
-- only be queried if redirects to them are found.
-- iana is not a whois db and does not need hostname
local whoisdb = {
arin = {
id = "arin",
hostname = "whois.arin.net", preflag = "+ ", postflag = "",
longname = {"american registry for internet numbers"},
fieldreq = fields_meta.arin,
smallnet_rule = fields_meta.arin.ob_netnum.netrange,
redirects = {
{"ob_org", "orgname", "longname"},
{"ob_org", "orgname", "id"},
{"ob_org", "orgid", "id"} },
output_short = {
{"ob_netnum", {"netrange", "netname"}},
{"ob_org", {"orgname", "orgid", {"country", "stateprov"}}} },
output_long = {
{"ob_netnum", {"netrange", "netname"}},
{"ob_org", {"orgname", "orgid", {"country", "stateprov"}}},
{"ob_cust", {"custname", {"country", "stateprov"}}},
{"ob_persn", {"orgtechname", "orgtechemail"}} },
reg = "netrange"},
ripe = {
id = "ripe",
hostname = "whois.ripe.net", preflag = "-B ", postflag = "",
longname = {"ripe network coordination centre"},
fieldreq = fields_meta.rpsl,
smallnet_rule = fields_meta.rpsl.ob_netnum.inetnum,
redirects = {
{"ob_role", "role", "longname"},
{"ob_org", "orgname", "id"},
{"ob_org", "orgname", "longname"} },
output_short = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}} },
output_long = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}},
{"ob_role", {"role", "email"}},
{"ob_persn", {"person", "email"}} },
reg = "inetnum" },
apnic = {
id = "apnic",
hostname = "whois.apnic.net", preflag = "", postflag = "",
longname = {"asia pacific network information centre"},
fieldreq = fields_meta.rpsl,
smallnet_rule = fields_meta.rpsl.ob_netnum.inetnum,
redirects = {
{"ob_netnum", "netname", "id"},
{"ob_org", "orgname", "longname"},
{"ob_role", "role", "longname"},
{"ob_netnum", "source", "id"} },
output_short = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}} },
output_long = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}},
{"ob_role", {"role", "email"}},
{"ob_persn", {"person", "email"}} },
reg = "inetnum" },
lacnic = {
id = "lacnic",
hostname = "whois.lacnic.net", preflag = "", postflag = "",
longname =
{"latin american and caribbean ip address regional registry"},
fieldreq = fields_meta.lacnic,
smallnet_rule = fields_meta.lacnic.ob_netnum.inetnum,
redirects = {
{"ob_netnum", "ownerid", "id"},
{"ob_netnum", "source", "id"} },
output_short = {
{"ob_netnum",
{"inetnum", "owner", "ownerid", "responsible", "country"}} },
output_long = {
{"ob_netnum",
{"inetnum", "owner", "ownerid", "responsible", "country"}},
{"ob_persn", {"person", "email"}} },
reg = "inetnum" },
afrinic = {
id = "afrinic",
hostname = "whois.afrinic.net", preflag = "-c ", postflag = "",
longname = {"african internet numbers registry",
"african network information center"},
fieldreq = fields_meta.rpsl,
smallnet_rule = fields_meta.rpsl.ob_netnum.inetnum,
redirects = {
{"ob_org", "orgname", "longname"} },
output_short = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}} },
output_long = {
{"ob_netnum", {"inetnum", "netname", "descr", "country"}},
{"ob_org", {"orgname", "organisation", "descr", "email"}},
{"ob_role", {"role", "email"}},
{"ob_persn", {"person", "email"}} },
reg = "inetnum" },
jpnic = {
id = "jpnic",
hostname = "whois.nic.ad.jp", preflag = "", postflag = "/e",
longname = {"japan network information center"},
fieldreq = fields_meta.jpnic,
output_short = {
{"ob_netnum", {"inetnum", "netname", "orgname"}} },
reg = "inetnum" },
iana = { -- not actually a db but required here
id = "iana", longname = {"internet assigned numbers authority"} }
}
-- cached results
nmap.registry.whois = {}
-- queue for queries
local queue = {}
for i,v in pairs(whoisdb) do
if i ~= "iana" then queue[i]={} end
end
local m_dotted =
"(%d+%.%d+%.%d+%.%d+)[%s]*[-][%s]*(%d+%.%d+%.%d+%.%d+)"
local m_cidr = "(%d+)[.]*(%d*)[.]*(%d*)[.]*(%d*)[/]+(%d+)"
local m_redir = {
"\n%s*[%S]%s*(%w*)%s*[Rr]esource",
"%s*(%w*)%s*[Ww]hois"
}
local m_none = {
"\n%s*([Nn]o match found for[%s\+]*$addr)",
"\n%s*([Uu]nallocated resource:%s*$addr)",
"\n%s*([Rr]eserved:%s*$addr)",
"\n[^\n]*([Nn]ot%s[Aa]ssigned[^\n]*$addr)",
"\n%s*(No match!!)%s*\n"
}
local m_err = {
"\n%s*([Aa]n [Ee]rror [Oo]ccured)%s*\n",
"\n[^\n]*([Ee][Rr][Rr][Oo][Rr][^\n]*)\n"
}
--
-- FUNCTIONS
--
-- trim
-- trim, as well as turn empty string into nil
local function trim(s)
if not s then return nil end
local td = (string.gsub(s, "^%s*(.-)%s*$", "%1"))
if td == "" then td = nil end
return td
end
-- have_asked
-- matches a string to elements of a table and returns false only if both
-- string and table are present and if no match is found. used to find
-- out whether a whoisdb has been queried before
local have_asked = function(pattern, t, dbg)
local s = ""
if t then s = table.concat(t, " ") end
if pattern and s then
if not s:match(pattern) then return false end
stdnse.print_debug(3, dbg .. " " .. pattern ..
" is one of the following dbs to have been queried: " .. s)
end
return true
end
-- querydo
-- connect to db, send query (host.ip), receive and return any results
local querydo = function(db, host)
local status, line
local result = ""
local socket = nmap.new_socket()
local catch = function()
stdnse.print_debug(id .. " " .. host.ip ..
" connection aborted, check the details for whoisdb." .. db)
pop_queue(host.ip)
socket:close()
end
local try = nmap.new_try(catch)
local v = whoisdb[db]
if not v then return false end
socket:set_timeout(10000)
connect_attempts = connect_attempts + 1
try(socket:connect(v.hostname, 43))
if check_registry(host.ip) then socket:close(); return end
send_attempts = send_attempts + 1
try(socket:send(v.preflag .. host.ip .. v.postflag .. "\n"))
if check_registry(host.ip) then socket:close(); return end
while true do
local status, lines = socket:receive_lines(1)
if not status then
break
else
result = result .. lines
if check_registry(host.ip) then socket:close(); return end
end
end
socket:close()
results_returned = results_returned + 1
return result
end
-- todotted
-- convert dword into dotted IPv4 notation (returns a string)
todotted = function(dword)
-- perhaps ought to check dword is legal
local a,b,c,_
a, _ = math.modf(dword/256^3)
dword = dword - ipOps.todword(a .. ".0.0.0")
b, _ = math.modf(dword/256^2)
dword = dword - ipOps.todword("0." .. b .. ".0.0")
c, _ = math.modf(dword/256)
dword = dword - ipOps.todword("0.0." .. c .. ".0")
return a .. "." .. b .. "." .. c .. "." .. dword
end
-- two_dwords
-- returns the two ip addresses at either end of a net range, as dwords
local two_dwords = function(str, patt)
local a, b, c, d, e, lo_net, host
a, b, c, d, e = str:match(patt)
local ipt = {b, c, d}
local strip = ""
for _, cap in ipairs(ipt) do
if cap == "" then cap = "0" end
strip = strip .. "." .. cap
end
lo_net = a .. strip
if e ~= "" then e = tonumber(e)
if e and e <=32 then
host = 32 - e end
end
return ipOps.todword(lo_net), ipOps.todword(lo_net) + 2^host - 1
end
-- smallest_net
-- to sort a table of net ranges by range, ascending
-- return true if net_1 is smaller than net_2
local smallest_net = function(net_1, net_2)
local sorted = true -- return value defaulting true to avoid a loop
local n1_lo, n1_hi, n2_lo, n2_hi
if net_1:match(m_dotted) then
n1_lo, n1_hi = net_1:match(m_dotted)
n1_lo = ipOps.todword(n1_lo)
n1_hi = ipOps.todword(n1_hi)
elseif net_1:match(m_cidr) then
n1_lo, n1_hi = two_dwords(net_1, m_cidr)
end
if net_2:match(m_dotted) then
n2_lo, n2_hi = net_2:match(m_dotted)
n2_lo = ipOps.todword(n2_lo)
n2_hi = ipOps.todword(n2_hi)
elseif net_2:match(m_cidr) then
n2_lo, n2_hi = two_dwords(net_2, m_cidr)
end
if n1_lo <= n2_lo and n1_hi >= n2_hi then sorted = false end
return sorted
end
-- ip_in_net
-- to determine if a given ip addr falls inside a given net range
-- return true if net contains ip
local ip_in_net = function(ip, net)
local i, j, net_lo, net_hi, dw_ip
if net:match(m_dotted) then
net_lo, net_hi = net:match(m_dotted)
net_lo = ipOps.todword(net_lo)
net_hi = ipOps.todword(net_hi)
elseif net:match(m_cidr) then
net_lo, net_hi = two_dwords(net, m_cidr)
end
dw_ip = ipOps.todword(ip)
if net_lo <= dw_ip and dw_ip <= net_hi then return true end
return false
end
-- find the smallest inetnum that includes host.ip and constrain result
-- to any information after the start of the smallest inetnum and before
-- any further inetnums
local constrain_result = function(result, thisdb, host)
local strbgn = 1
local strend = 1
local mptr = {}
local mstr = {}
local ptr = 1
local bound = nil
local dbg = id .. " " .. host.ip .. " " .. thisdb
-- find all inetnums
-- build two tables
while strbgn and whoisdb[thisdb].fieldreq do
strbgn, strend =
result:find(whoisdb[thisdb].fieldreq.ob_exist, strend)
if strbgn then
-- one consisting the available inetnum pointers
table.insert(mptr, strbgn)
-- one consisting the available inetnum strings
table.insert(mstr,
trim(result:match(whoisdb[thisdb].smallnet_rule, strbgn)))
end
end
if # mstr > 1 then
-- find the closest one to host.ip and constrain the result to it
stdnse.print_debug(3, dbg .. " Focusing on the smallest of " ..
#mstr .. " address ranges")
-- sort the table mptr into nets ascending
table.sort(mstr, smallest_net)
-- select the first net that includes host.ip
local str_net
for _, strnet in ipairs(mstr) do
if ip_in_net(host.ip,strnet) then
str_net = strnet
break
end
end
stdnse.print_debug(3, dbg ..
" smallest range containing target IP addr. is " ..
trim(str_net))
-- get all pointers and sort in order
-- this will be unpredictable if we have two equal ranges!
table.sort(mptr)
-- loop through pointers until our inetnum pointer matches
for i = #mptr,1,-1 do
if str_net ==
trim(result:match(whoisdb[thisdb].smallnet_rule, mptr[i]))
then
ptr = mptr[i]
-- check to see if there's a greater pointer for bounding
if mptr[i+1] then bound = mptr[i+1] end
break
end
end
local dbg =
dbg .. " smallest range is offset from " .. ptr .. " to"
-- isolate inetnum and associated objects
if bound then
stdnse.print_debug(3, dbg .. " " .. bound)
-- get from pointer to bound
return result:sub(ptr,bound), ptr
else
stdnse.print_debug(3, dbg .. " the end.")
-- or get the whole thing from the pointer onwards
return result:sub(ptr), ptr
end
end -- if # mstr
return result, 0
end -- function
-- EXTRACT_SOME_DATA
-- returns a table of objects or, a single object if that object is
-- passed as a fourth arg
local extract_some_data = function(rchunk, thisdb, host, obg)
local ret = {}
local dbg = id .. " " .. host.ip
local groups = {}
-- we either pass a table for one object group or for all groups
if obg then
groups[obg] = whoisdb[thisdb].fieldreq[obg]
stdnse.print_debug(3, dbg .. " " .. thisdb ..
" Extracting a single group of objects: " .. obg)
else
stdnse.print_debug(3, dbg .. " " .. thisdb ..
" Extracting all object groups.")
groups = whoisdb[thisdb].fieldreq
end
for object_group, ogt in pairs(groups) do
if object_group and object_group ~= "ob_exist" then
stdnse.print_debug(4, dbg .. " " .. thisdb ..
" matching object group " .. object_group)
ret[object_group] = {}
local pttn_grp = groups[object_group].ob_start
local pttn_grpend = groups[object_group].ob_end
local i, j, ob_start, ob_end
-- get a substr of rchunk that corresponds to an object group
ob_start, j = rchunk:find(pttn_grp)
i, ob_end = rchunk:find(pttn_grpend, j)
-- if we could not find the end, make the end EOF
ob_end = ob_end or -1
if ob_start and ob_end then
stdnse.print_debug(4, dbg .. " " .. thisdb ..
" capturing " .. object_group .. " with indices " ..
ob_start .. " " .. ob_end)
local obj_raw = rchunk:sub(ob_start,ob_end)
for fieldname, matchline in pairs(groups[object_group]) do
if fieldname ~= "ob_start"
and fieldname ~= "ob_end" then
ret[object_group][fieldname] =
trim(obj_raw:match(matchline))
end
end
end -- if ob_start and ob_end
end -- if object_group
end -- for object_group
if obg then ret = ret[obg] end -- returning one object
return ret
end -- function
-- RULESET
-- returns found, redirect, ianahits
-- found is true if no redirects detected, redirect is a member of whoisdb
-- ianahits is incremented here if a redirect to iana is detected
local ruleset = function(db, host, newwdata, ihits)
stdnse.print_debug(3, id .. " " .. host.ip .. " " .. db ..
" Testing for redirection.")
local redirect = nil -- the value of a recognised redirect
local found = false -- flag if we have found the record we seek
-- decide what action to take based on the presence of a redirect
-- returns three values akin to found, redirect, ihits
local decide_next_move = function(nom, i, db)
local iana = whoisdb.iana.id
local arin = whoisdb.arin.id
-- arin record points to iana so we won't follow and we assume we
-- have our record
if nom == iana and db == arin then
stdnse.print_debug(2, id .. " " .. host.ip .. " " .. db ..
" Accept arin record (matched IANA)")
return true, nil, (i + 1) end
-- other record points to iana so we query arin next
if nom == iana then
stdnse.print_debug(2, id .. " " .. host.ip .. " " .. db ..
" Redirecting to arin (matched IANA)")
return false, arin, (i + 1) end
-- redirect, but not to iana or to self, so we follow it.
if nom ~= whoisdb[db].id then
stdnse.print_debug(2, id .. " " .. host.ip .. " " .. db ..
" Redirecting to " .. nom)
return false, nom, i end
-- redirect to self
return true, nil, i
end
-- test for redirects
if whoisdb[db].redirects then
-- iterate over each table of redirect info for a specific field
for _, v in ipairs(whoisdb[db].redirects) do
local ob, fld, prop = unpack(v) -- three redirect elements
-- if a field has been captured for the given redirect info
if newwdata[db][ob] and newwdata[db][ob][fld] then
stdnse.print_debug(3, id .. " " .. host.ip .. " " .. db ..
" match redirect in object: " .. ob .."." .. fld ..
" for " .. prop)
-- iterate over the members of whoisdb to find a match
for member,mem_properties in pairs(whoisdb) do
-- check whether the field value captured contains the
-- string property of the current member of whoisdb.
-- if it does, we have a redirect and we have to
-- decide whether to follow the redirect
if type(mem_properties[prop]) == "string" then
if string.lower(newwdata[db][ob][fld])
:match(mem_properties[prop]) then
stdnse.print_debug(3, id .. " " .. host.ip ..
" " .. db .. " matched " .. prop ..
" in " .. ob .."." .. fld)
return decide_next_move(whoisdb[member].id,
ihits, db)
end
elseif type(mem_properties[prop]) == "table" then
-- tables are used for properties with more than
-- one value, iterate over each value which should
-- be of type string
for _, m in ipairs(mem_properties[prop]) do
if string.lower(newwdata[db][ob][fld])
:match(m) then
stdnse.print_debug(3, id .. " " ..
host.ip .. " " .. db .. " matched " ..
prop .. " in " .. ob .."." .. fld)
return
decide_next_move(whoisdb[member].id,
ihits, db)
end
end -- for _, m in ipairs
end
end -- for mem, mem_properties
end
end -- for _,v in ipairs
end -- if whoisdb
-- if redirects have not been found then assume that the record has
-- been found.
found = true
return found, redirect, ihits
end
function check_registry(ip, db)
if not next(nmap.registry.whois) then return nil end
local h_ip = ip
for cached_ip, t in pairs(nmap.registry.whois) do
if ip ~= cached_ip and t.range and ip_in_net(ip,t.range) and
math.abs(ipOps.todword(h_ip) - ipOps.todword(cached_ip)) < 256 then
if t.next_thisdb then
-- pop_queue(ip,db)
return cached_ip, t.next_thisdb
else
pop_queue(ip)
return cached_ip, nil
end
end
end
end
function pop_queue(ip, db)
-- remove host.ip from db queue or, all queues if db is nil
if not db then
for db,_ in pairs(queue) do
for i,_ in ipairs(queue[db]) do
if queue[db][i] == ip then
table.remove(queue[db],i)
break
end
end
end
else
for i,_ in ipairs(queue[db]) do
if queue[db][i] == ip then
table.remove(queue[db],i)
break
end
end
end
end
-- create a single socket object per instance for sleep() hack
local pcap = nmap.new_socket()
function pcap_callback() return 'xxx' end
--
--
-- ACTION
--
action = function(host, port)
-- open pcap socket for sleep() hack (if not open already)
if not pcap:get_info() then
pcap:pcap_open(host.interface, 64, 0, pcap_callback,
"tcp[0:2] == 0xFFFF and tcp[2:2] == 0xFFFF" )
end
-- add host to the queues for each whoisdb
for i,v in pairs(queue) do
table.insert(queue[i], host.ip)
end
-- expect liberal checking for cached results!
local cached_ip, next_thisdb = check_registry(host.ip)
if cached_ip and not next_thisdb then return "see the result for " ..
cached_ip end
local nextdb = next_thisdb or nil
-- Handle commandline args
if nmap.registry.args then
local cmd
local t = {}
if nmap.registry.args.whois and nmap.registry.args.whois.whodb
then
cmd = nmap.registry.args.whois.whodb end
if nmap.registry.args.whodb then
cmd = nmap.registry.args.whodb end
if cmd then
for db in string.gmatch(cmd, "%w+") do
if not whoisdb[db] then else
t[#t+1] = db
end
end
if #t > 0 then
orderdefn = t
stdnse.print_debug(2, id .. " " .. host.ip ..
" script args: " .. table.concat(t, " "))
end
end
end
local _, tmp -- local temp throw-away vars
local trynum = 0 -- incr. for each qry to dbs in
orderdefn
local numtotry = # orderdefn -- total number of dbs in orderdefn
local thisdb = "" -- the currently queried db
local result -- thisdb response to our query
local found = false -- true when record has been found
local ianahits = 0 -- # times a record has pointed to IANA
local stuff_to_display = nil -- the output
local have_objects = false -- true when we recognise the response
local track = {} -- keep track of which dbs queried
local extrctd_data = {} -- table of extracted data
local verbos_level = nmap.verbosity() --Verbosity
local dbg = id .. " " .. host.ip
--
-- Send queries to whois db servers and analyse responses...
--
-- stop queries when record is found or we exhaust list of whois
-- servers defined in orderdefn
while trynum <= numtotry and not found do
-- try all whois dbs defined by orderdefn (in order defined)
-- except when specifically redirected elsewhere
if not nextdb then
-- thisdb is a db defined in orderdefn
if trynum >= numtotry then break end
trynum = trynum + 1
thisdb = orderdefn[trynum]
else
-- thisdb is a redirect
thisdb = nextdb
end
-- check whether we have already queried thisdb
stdnse.print_debug(3, dbg .. " Have we already queried " ..
thisdb .. "?" )
local prevqd = have_asked(thisdb, track, dbg)
if prevqd and nextdb then
-- We assume a redirect back to this db means record is found.
stdnse.print_debug(2, dbg .. " " .. thisdb ..
" has been queried previously!")
found = true
break
end
local alt = ''
if not nextdb then
-- if we have asked orderdefn[trynum] already, try trynum+1
-- until we run out of dbs in orderdefn
while prevqd and (trynum < numtotry) do
trynum = trynum + 1
thisdb = orderdefn[trynum]
prevqd = have_asked(thisdb, track, dbg)
end
-- we've exhausted our list of whoisdbs to try
if prevqd then
stdnse.print_debug(2, dbg ..
" All whois dbs defined have been queried.")
break
end
else
-- debug
alt = " (from redirect)"
end
-- Check the registry
cached_ip, next_thisdb = check_registry(host.ip,thisdb)
if cached_ip and not next_thisdb then return "see the result for " ..
cached_ip end
thisdb = next_thisdb or thisdb
-- Make this coroutine wait until it is first in the queue for thisdb
local count_loops = 1
local time_out = 500
local first_in_queue = queue[thisdb][1]
while queue[thisdb][1] ~= host.ip do
-- reset count if the first_in_queue has changed during this loop
if queue[thisdb][1] and queue[thisdb][1] ~= first_in_queue then
for cached_ip, t in pairs(nmap.registry.whois) do
if cached_ip and t.range and not
ip_in_net(queue[thisdb][1],t.range) then
first_in_queue = queue[thisdb][1]
count_loops = 1
break
end
end
end
-- yield() this coroutine for time_out ms
pcap:set_timeout(time_out)
pcap:pcap_register('aaa')
_, _, _, _, _ = pcap:pcap_receive()
count_loops = count_loops + 1
cached_ip, next_thisdb = check_registry(host.ip,thisdb)
-- if a result for the range containing host.ip has been found,
-- return immediately from action().
if cached_ip and not next_thisdb then
return "see the result for " .. cached_ip end
-- if a redirect has been found for the range containing host.ip,
-- queue instead for next_thisdb.
if next_thisdb and next_thisdb ~= thisdb then
thisdb = next_thisdb; count_loops = 1 end
end
--
-- SEND QUERY TO THISDB
--
stdnse.print_debug(2, dbg .. " Begin Query at " .. thisdb .. alt)
result = querydo(thisdb, host)
-- Check the registry
local cached_ip, this_nextdb = check_registry(host.ip)
if cached_ip and not this_nextdb then return "see the result for " ..
cached_ip end
-- keep track of which dbs we've queried - add thisdb
track[#track+1] = thisdb
-- break out of the main loop if we have unknown issues...
if not result and not this_nextdb then
stdnse.print_debug(1, dbg .. " " .. thisdb ..
" RESULT IS FALSE! Eh?")
break
end
local result_chunk = result
-- do we recognise objects in the result?.
if whoisdb[thisdb].fieldreq and not next_thisdb then
have_objects = result:match(whoisdb[thisdb].fieldreq.ob_exist)
elseif not whoisdb[thisdb].fieldreq then
have_objects = false
stdnse.print_debug(2, "missing property: whoisdb." ..
thisdb .. ".fieldreq")
elseif next_thisdb then
nextdb = next_thisdb
have_objects = false
found = false
end
-- if we do not recognise objects check for an error/message
if not have_objects and not next_thisdb then
nextdb = nil
stdnse.print_debug(2, dbg .. " " .. thisdb ..
" has not responded with the expected objects")
local msg
-- We may have a redirect
for _, pttn in ipairs(m_redir) do
for tmp in result:gfind(pttn) do
if tmp and string.lower(tmp) ~= "iana" and
string.lower(tmp) ~= thisdb and
whoisdb[string.lower(tmp)] then
nextdb = string.lower(tmp)
break
end
end
if nextdb then break end
end
-- may have found our record saying something similar to
-- "No Record Found"
for _, pttn in ipairs(m_none) do
pttn = pttn:gsub("$addr", host.ip)
tmp = result:match(pttn)
if tmp then
msg = tmp
stdnse.print_debug(2, dbg .. " " .. thisdb ..
" responded with a message which is assumed to be authoritative.")
found = true
break
end
end
-- We may have an error
if not msg then
for _, pttn in ipairs(m_err) do
tmp = result:match(pttn)
if tmp then
msg = tmp
stdnse.print_debug(2, dbg .. " " .. thisdb ..
" responded with an ERROR message.")
break
end
end
end
-- if we've recognised a non-object message,
if msg then
stuff_to_display = "Message from " ..
whoisdb[thisdb].hostname .. "\n" .. msg
end
end
-- the query result may not contain the set of objects we were
-- expecting and we do not recognise the response message.
-- it may contain a record mirrored from a different whois db
if not nextdb and not have_objects and not found then
local db_mirrored
for setname, set in pairs(fields_meta) do
if set ~= whoisdb[thisdb].fieldreq and
result:match(set.ob_exist) then
db_mirrored = setname
stdnse.print_debug(1, dbg .. " " .. thisdb ..
" seems to have responded using the set of objects named: " ..
setname)
break
end
end
if db_mirrored then
-- find a display to match the objects.
-- this probably needs to be less random
for some_db, db_props in pairs(whoisdb) do
if db_props.fieldreq and whoisdb[db_mirrored].fieldreq and
db_props.fieldreq == whoisdb[db_mirrored].fieldreq then
whoisdb[thisdb].redirects = nil
whoisdb[thisdb].fieldreq = whoisdb[some_db].fieldreq
whoisdb[thisdb].smallnet_rule = whoisdb[some_db].smallnet_rule
whoisdb[thisdb].output_short = whoisdb[some_db].output_short
whoisdb[thisdb].output_long = whoisdb[some_db].output_long
have_objects = true
stdnse.print_debug(1, dbg .. " " .. thisdb ..
" will use the display properties of " .. some_db)
break
end
end
end -- if db_mirrored
end
-- extract fields from the entire result, do redirect discovery.
if have_objects then
extrctd_data[thisdb] = extract_some_data(result, thisdb, host)
end
if have_objects and whoisdb[thisdb].redirects then
found, nextdb, ianahits =
ruleset(thisdb, host, extrctd_data, ianahits)
if not found and nextdb then
-- cache this wide range and redirect
if extrctd_data[thisdb] and extrctd_data[thisdb].ob_netnum then
nmap.registry.whois[host.ip] = {}
nmap.registry.whois[host.ip].range =
extrctd_data[thisdb].ob_netnum[whoisdb[thisdb].reg]
nmap.registry.whois[host.ip].next_thisdb = nextdb
-- remove all ips in cached range from the queue
for cached_ip, t in pairs(nmap.registry.whois) do
if host.ip == cached_ip then
local h_ip = queue[thisdb][1]
while h_ip and t.range and ip_in_net(h_ip,t.range) and
math.abs(ipOps.todword(cached_ip) - ipOps.todword(h_ip))
< 256 do
pop_queue(queue[thisdb][1],thisdb)
h_ip = queue[thisdb][1]
end
end
end
end
end
else
found, nextdb = true, nil
end
if have_objects and found then
-- optionally constrain result to a more focused area
-- discarding previous extraction
if whoisdb[thisdb].smallnet_rule then
local offset, ptr, strbgn, strend
result_chunk, offset = constrain_result(result, thisdb, host)
if offset > 0 then
extrctd_data[thisdb] = nil
extrctd_data[thisdb] =
extract_some_data(result_chunk, thisdb, host)
end
if offset > 1 then
-- fetch an object immediately in front of inetnum
stdnse.print_debug(2, dbg .. " " .. thisdb ..
" Searching for an object group immediately before this range.")
result_chunk = result:sub(1,offset)
-- avoid getting the first few objects of the result by starting
-- half-way between 1 and offset which should be enough...
ptr = (math.modf((result_chunk:len())/2))
for i, v in pairs(whoisdb[thisdb].fieldreq) do
if i ~= "ob_exist" then
strbgn, strend =
result_chunk:find(v.ob_start, ptr)
if strbgn then
if not next(extrctd_data[thisdb][i]) then
tmp, strend = result_chunk:find(v.ob_end, ptr +
strbgn)
strend = strend or -1
extrctd_data[thisdb][i] = extract_some_data(
result_chunk:sub(strbgn, strend), thisdb, host,
i)
end
end
end
end -- for i, v
end -- if offset
end -- if whoisdb[thisdb].smallnet_rule
-- DEBUG
stdnse.print_debug(4, dbg .. " " .. thisdb ..
" Fields captured :")
for obgroup, t in pairs(extrctd_data[thisdb]) do
for fieldname, fieldvalue in pairs(t) do
stdnse.print_debug(4, dbg .. " " .. thisdb .. " field " ..
obgroup .. "." ..fieldname .. " " .. fieldvalue)
end
end
end -- if have_objects and found
-- DEBUG
if nextdb then stdnse.print_debug(2, dbg .. " " .. thisdb ..
" redirects to " .. nextdb) end
local dbg_found =
dbg .. " " .. thisdb .. " query concluded with Record "
if found then
dbg_found = dbg_found .. "Found"
else
dbg_found = dbg_found .. "Not Found"
end
stdnse.print_debug(2, dbg_found)
-- if cache exists we can remove host.ip from the queue for thisdb
pop_queue(host.ip, thisdb)
end -- WHILE trynum <= numtotry and not found
-- ALL QUERIES COMPLETED
-- CATCH IANA
-- if we have not found a good record but had more than one redirect
-- to iana then default to the arin record
if not found and ianahits > 1 then
stdnse.print_debug(1, dbg ..
" A Record not been found, but we have been directed to IANA " ..
ianahits .. " times.")
found = true
thisdb = whoisdb.arin.id
end
-- NO RECORD HAS BEEN FOUND/RECOGNISED
if not found then
-- remove host.ip from all queues
pop_queue(host.ip)
return "script failed to find a record or " ..
"couldn't understand the query responses."
end
-- CACHE A RESULT IN THE REGISTRY
-- store the target IP address and net range
if found and have_objects then
if extrctd_data[thisdb] and extrctd_data[thisdb].ob_netnum then
nmap.registry.whois[host.ip] = {}
nmap.registry.whois[host.ip].range =
extrctd_data[thisdb].ob_netnum[whoisdb[thisdb].reg]
end
end -- if found and have_objects
if found and not have_objects and stuff_to_display then
-- We don't have a net range to cache so we create one using the
-- (poss. inacurate) assumption that any target within 32 addresses
-- will share the same non-record message
local fake_range = host.ip .. " - " ..
todotted(ipOps.todword(host.ip) + 31)
nmap.registry.whois[host.ip] = {}
nmap.registry.whois[host.ip].range = fake_range
for cached_ip, t in pairs(nmap.registry.whois) do
if host.ip == cached_ip then
local h_ip = queue[thisdb][1]
while h_ip and t.range and ip_in_net(h_ip,t.range) and
math.abs(ipOps.todword(cached_ip) - ipOps.todword(h_ip))
< 256 do
pop_queue(queue[thisdb][1],thisdb)
h_ip = queue[thisdb][1]
end
end
end
end
-- We can now remove host.ip from each queue it's in
pop_queue(host.ip)
-- DISPLAY THE FOUND RECORD
-- ipairs over the table that dictates the order in which fields
-- should be output
local grpname = ""
local fieldname = ""
local display_rules = {}
if whoisdb[thisdb].output_long and
not whoisdb[thisdb].output_short then
whoisdb[thisdb].output_short = whoisdb[thisdb].output_long
end
if have_objects and whoisdb[thisdb].output_short then
if verbos_level > 0 and whoisdb[thisdb].output_long then
display_rules = whoisdb[thisdb].output_long
else
display_rules = whoisdb[thisdb].output_short
end
stuff_to_display = "Record found at " .. whoisdb[thisdb].hostname
for _, obtbl in ipairs(display_rules) do
for _, flds in ipairs(obtbl) do
if type(flds) == "string" then
grpname = flds
if not extrctd_data[thisdb][grpname] then break end
end
if type(flds) == "table" then
for _, fld in ipairs(flds) do
local fvalue
if type(fld) == "string" then
fvalue = extrctd_data[thisdb][grpname][fld]
if fvalue then
stuff_to_display = stuff_to_display .. " \n" .. fld ..
": " .. fvalue
end
end
if type(fld) == "table" then
local buildstr
local f
for _, fd in ipairs(fld) do
fvalue =
extrctd_data[thisdb][grpname][fd]
if fvalue then
if not buildstr then
buildstr = fd .. ": " .. fvalue
else
buildstr = buildstr .. " " .. fd .. ":
" .. fvalue
end
end
end
if buildstr then
stuff_to_display = stuff_to_display ..
" \n" .. trim(buildstr)
end
end
end -- for _, fld
end
end -- for _, flds
end -- for _, obtbl
elseif not stuff_to_display then
stuff_to_display = " Record was not found."
end -- if have_objects and
-- alles klar
return stuff_to_display
end -- ACTION
_______________________________________________ Sent through the nmap-dev mailing list http://cgi.insecure.org/mailman/listinfo/nmap-dev Archived at http://SecLists.Org
Current thread:
- Re: [NSE] WHOIS - Attempts to queue coroutines to limit the number of whois queries. Fyodor (Apr 05)
- <Possible follow-ups>
- Re: [NSE] WHOIS - Attempts to queue coroutines to limit the number of whois queries. majek04 (Apr 08)
- Re: [NSE] WHOIS - Now with Queuing and Cached Results. jah (Apr 22)
- Re: [NSE] WHOIS - Now with Queuing and Cached Results. Fyodor (Jun 13)
- Re: [NSE] WHOIS - Now with Queuing and Cached Results. jah (Jun 14)
- Re: [NSE] WHOIS - Now with Queuing and Cached Results. Patrick Donnelly (Jun 14)
- Re: [NSE] WHOIS - Now with Queuing and Cached Results. jah (Apr 22)