Re: [NSE] Web application detection

[Sven Klemm <sven () c3d2 de>]

Diman Todorov wrote:
> this brings up an interesting point. Have you had a look at the  
> nselib? So far it has a very small dynamically loadable module  
> written in c for bitwise operations which serves as a concept  
> demonstration. Do you think you could easily make a loadable NSE  
> module out of your libxml2 wrapper? I could imagine that it would  
> only be compiled if users have libxml2 installed.

The attached patch adds it as a new library. I left out the libxml2
detection stuff as I don't know how to do that.
The only thing that is currently wrapped is parsing html and xml
documents and querying those parsed documents with xpath.
I attached the sedusa.nse script again because I changed the case for
xml table to lowercase.

Cheers,
sven

Index: xml.c
===================================================================
--- xml.c       (revision 0)
+++ xml.c       (revision 0)
@@ -0,0 +1,121 @@
+
+#include "xml.h"
+
+#include <libxml/HTMLparser.h>
+#include <libxml/xpath.h>
+
+typedef struct XmlDocumentData {
+  xmlDocPtr document;
+} XmlDocumentData;
+
+// create a lua XmlDocument
+int create_document( lua_State * L, xmlDocPtr doc ) {
+  if ( doc ) {
+    // parsing successful
+    XmlDocumentData * doc_data;
+    doc_data = (XmlDocumentData *) lua_newuserdata( L, sizeof(XmlDocumentData));
+    // set metatable for userdata
+    luaL_getmetatable( L, "XmlDocument" );
+    lua_setmetatable( L, -2 );
+    doc_data->document = doc;
+  } else  {
+    // parsing failed
+    luaL_error( L , "parsing document failed." );
+  }
+  return 1;
+}
+
+// takes an html document as string and returns a XmlDocument
+int xml_parse_html( lua_State * L ) {
+  const char * doc_string = luaL_checkstring( L, 1 );
+  lua_pop(L, 1);
+  char * url = NULL;
+  char * encoding = NULL;
+
+  int options = HTML_PARSE_RECOVER | HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_NONET;
+  htmlDocPtr doc = htmlReadDoc( doc_string, url, encoding, options );
+
+  return create_document( L, doc );
+}
+
+// takes an xml document as string and returns a XmlDocument
+int xml_parse_xml( lua_State * L ) {
+  const char * doc_string = luaL_checkstring( L, 1 );
+  lua_pop(L, 1);
+  char * url = NULL;
+  char * encoding = NULL;
+
+  int options = XML_PARSE_RECOVER | XML_PARSE_NOERROR | XML_PARSE_NOWARNING | XML_PARSE_NONET;
+  xmlDocPtr doc = xmlReadDoc( doc_string, url, encoding, options );
+
+  return create_document( L, doc );
+}
+
+
+static const struct luaL_reg xml_methods[] = {
+  { "parse_html", xml_parse_html },
+  { "parse_xml", xml_parse_xml },
+  { NULL, NULL }
+};
+
+// takes an xpath expression and return the match(es) as string(s)
+int xmldoc_xpath( lua_State * L ) {
+  XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L, 1, "XmlDocument");
+  const char * xpath = luaL_checkstring( L, 2 );
+  xmlXPathContextPtr context = xmlXPathNewContext( doc->document );
+
+  if ( !context ) luaL_error( L, "Error creating context." );
+
+  xmlXPathObjectPtr result = xmlXPathEvalExpression(xpath, context);
+  xmlXPathFreeContext(context);
+  if ( !result ) luaL_error( L, "Error while evaluating XPath expression." );
+
+  if( xmlXPathNodeSetIsEmpty( result->nodesetval ) ) {
+    // empty resultset
+    lua_pushnil( L );
+    xmlXPathFreeNodeSetList( result );
+    return 1;
+  } else {
+    int i;
+    // we found something
+    xmlNodeSetPtr nodeset = result->nodesetval;
+    for ( i = 0; i < nodeset->nodeNr; i++) {
+      char * tmp = xmlXPathCastNodeToString( nodeset->nodeTab[i] );
+      lua_pushstring( L, tmp );
+      free( tmp );
+    }
+    int matches = nodeset->nodeNr;
+    xmlXPathFreeNodeSetList( result );
+    return matches;
+  }
+}
+
+int xmldoc_free( lua_State * L ) {
+  XmlDocumentData * doc = (XmlDocumentData *) luaL_checkudata(L, 1, "XmlDocument");
+  free( doc->document );
+  return 0;
+}
+
+// XmlDocument methods
+static const struct luaL_reg xmldoc_methods[] = {
+  { "xpath", xmldoc_xpath },
+  { "__gc", xmldoc_free },
+  { NULL, NULL }
+};
+
+
+// initializer function, called when library is required
+int luaopen_xml( lua_State * L ) {
+
+  // create metatable
+  luaL_newmetatable( L, "XmlDocument" );
+  // metatable.__index = metatable
+  lua_pushvalue( L, -1 );
+  lua_setfield( L, -2, "__index" );
+  // register methods
+  luaL_register( L, NULL, xmldoc_methods );
+
+  luaL_register( L, "xml", xml_methods );
+  return 1;
+}
+
Index: Makefile.in
===================================================================
--- Makefile.in (revision 5165)
+++ Makefile.in (working copy)
@@ -12,15 +12,20 @@
 LIBTOOL= ./libtool
 LTFLAGS = --tag=CC --silent
 
-all: bit.so
+all: bit.so xml.so
 
 bit.so: bit.c @LIBTOOL_DEPS@
        $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $(CFLAGS) -c bit.c
        $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -avoid-version -module -rpath /usr/local/lib -o bit.la bit.lo
        mv .libs/bit.so bit.so
 
+xml.so: xml.c @LIBTOOL_DEPS@
+       $(LIBTOOL) $(LTFLAGS) --mode=compile $(CC) @LUAINCLUDE@ $(LIBXML_INCLUDE) $(CFLAGS) -c xml.c
+       $(LIBTOOL) $(LTFLAGS) --mode=link $(CC) -lxml2 -avoid-version -module -rpath /usr/local/lib -o xml.la xml.lo
+       mv .libs/xml.so xml.so
+
 clean: 
-       rm -f bit.so *.la *.lo
+       rm -f bit.so xml.so *.la *.lo
        rm -rf .libs
 
 distclean: clean
Index: xml.h
===================================================================
--- xml.h       (revision 0)
+++ xml.h       (revision 0)
@@ -0,0 +1,13 @@
+
+#ifndef XMLLIB
+#define XMLLIB
+
+#define XMLLIBNAME "xml"
+
+#include "lauxlib.h"
+#include "lua.h"
+
+LUALIB_API int luaopen_xml(lua_State *L);
+
+#endif
+
Index: nselib.h
===================================================================
--- nselib.h    (revision 5165)
+++ nselib.h    (working copy)
@@ -2,6 +2,7 @@
 #define NSE_LIB
 
 #define NSE_BITLIBNAME "bit"
+#define NSE_XMLLIBNAME "xml"
 
 
 #endif

id = "Sedusa"
description = "Connects to an HTTP server and tries to guess the running web application."
author = "Sven Klemm <sven () c3d2 de>"
license = "See nmaps COPYING for licence"
categories = {"safe"}

require "stdnse"
require "shortport"
require "url"
require "xml"

portrule = shortport.service({'http', 'https', 'ssl/http'})

action = function(host, port)
  local scheme, hostname, app, exps, index, xpath, u, doc
  local target = {}

  if port.service == 'https' or port.version.service_tunnel == 'ssl' then
    target.scheme = "https"
    if port.number ~= 443 then target.port = port.number end
  else
    target.scheme = "http"
    if port.number ~= 80 then target.port = port.number end
  end

  if (host.name and not host.name == "") then
    target.host = host.name
  else
    target.host = host.ip
  end
  target.path = "/"

  u = url.build( target )
  doc = Sedusa.get_document( u )

  for app, exps in pairs( Sedusa.hints ) do
    for index, xpath in pairs( exps ) do
      if doc.xml:xpath( xpath ) then
        if Sedusa.verify[app] then
          local version = Sedusa.verify[app]( u )
          if version then
            return app .. " " .. version
          else
            return app .. " version not identified."
          end
        else
          return "No verify function for " .. app .. " found."
        end
        break
      end
    end
  end
  
end

Sedusa = {

  hints = {},
  verify = {},

  document_cache = {},

  Document = {
    new = function( h, b )
      local parsed = nil
      if string.len( b ) > 0 then
        parsed = xml.parse_html( b )
      end
      return { header = h, body = b, xml = parsed }
    end
  },

  get_document = function( url )
    local document

    if not Sedusa.document_cache[ url ] then
      document = Sedusa.http_get( url )
      Sedusa.document_cache[ url ] = document
    else
      document = Sedusa.document_cache[ url ]
    end
    if document.header['Status'] == 301 or
       document.header['Status'] == 302 
     then
      document = Sedusa.get_document( document.header['Location'] )
    end

    return document
  end,

  http_get = function( u )
    local protocol, port, request, query, socket
    local parsed = url.parse( u )
    
    if parsed.scheme == 'https' then
      protocol = "ssl"
      port = 443
    else
      protocol = "tcp"
      port = 80
    end
  
    if ( parsed.port ) then port = parsed.port end
  
    query = parsed.path
    if ( parsed.query ) then
      query = query .. '?' .. parsed.query 
    end
  
    request = "GET "..query.." HTTP/1.1\r\nHost: "..parsed.host.."\r\n\r\n"
  
    socket = nmap.new_socket()
    socket:connect( parsed.host, port, protocol )
    socket:send(request)
  
    local buffer = stdnse.make_buffer( socket, "\r?\n")
  
    local status, line, key, value, head, header
    head, header = {}, {}
    -- head loop
    while true do
      status, line = buffer()
      if (not status or line == "") then break end
      table.insert(header,line)
    end
  
    -- build nicer table for header
    for key, value in pairs( header ) do
      if key == 1 then
        local code = select( 3, string.find( value, "HTTP/%d\.%d (%d+)") )
        head['Status'] = tonumber(code)
      else
        key, value = select( 3, string.find( value, "(.+): (.*)" ) )
        if key and value then
          head[key] = value:gsub( '[\r\n]+', '' )
        end
      end
    end
  
    local body = {}
    while true do
      status, line = buffer()
      if (not status) then break end
      table.insert(body,line)
    end
  
    socket:close()
    return Sedusa.Document.new( head, table.concat(body) )
  end,

}



-- setup default detector for some web applications
for key, app in pairs( {"b2evolution", "bBlog", "C3D2-Web", "DokuWiki", "gitweb", "Midgard", "Pentabarf", "PhpWiki", 
"Plone", "PostNuke", "TYPO3", "vBulletin"} ) do
  Sedusa.hints[app] = { '//meta[@name="generator" and starts-with(@content,"' .. app .. '")]' }
  Sedusa.verify[app] = function( url ) 
    local document = Sedusa.get_document( url )
    -- look in generator meta tag
    local generator = document.xml:xpath( '//meta[@name="generator"]/@content' )
    if generator and string.match( generator, app .. " (.*)" ) then
      return string.match( generator, app .. " (.*)" )
    end
  end

end

Sedusa.verify["PhpWiki"] = function( url )
  local document = Sedusa.get_document( url )

  local generator = document.xml:xpath( '//meta[@name="PHPWIKI_VERSION"]/@content' )
  if generator then
    return generator
  end

end


Sedusa.hints["Drupal"] = {
  '//div[@id="block-user-0" and h2[text()="User 
login"]]/div[@class="content"]/form[@id="user-login-form"]/div[div[@class="form-item"]/input[@type="text" and 
@class="form-text required"]]',
}

Sedusa.hints["Joomla!"] = {
  '//meta[@name="Generator" and starts-with(@content,"Joomla!")]'
}

Sedusa.hints["MediaWiki"] = { 
  '//body[contains(@class, "mediawiki")]',
  '//div[@id="footer"]/div[@id="f-poweredbyico"]/a[@href="http://www.mediawiki.org/"]/img&apos;
}

Sedusa.verify["MediaWiki"] = function( u )
  local document = Sedusa.get_document( u )
  local link = document.xml:xpath( '//a[contains(@href,"index.php?title=") and starts-with(@href, "/")]/@href' )
  if link then
    link = link:match( "^(.*/index.php[?]title=).*")
  else
    link = document.xml:xpath( '//script[@type="text/javascript" and contains(text(), "var wgScriptPath = ")]' )
    link = link:gsub( "[\r\n]", "" )
    if link then
      link = link:match( 'var wgScriptPath = "([^"]+)"') .. '/index.php?title='
    end
  end

  -- get version from special:version
  local version = Sedusa.get_document( url.absolute( u, link .. "Special:Version" ) )
  if version.xml:xpath('//div/ul/li[a[@href="http://www.mediawiki.org/"; and text()="MediaWiki"]]') then
    return string.match( version.xml:xpath('//div/ul/li[a[@href="http://www.mediawiki.org/"; and text()="MediaWiki"]]'), 
"MediaWiki: ([^\r\n]+)" )
  end

  -- get version from atom feed
  local atom = Sedusa.get_document( url.absolute( u, link .. "Special:Recentchanges&feed=atom" ) )
  if atom.xml:xpath( '//generator[starts-with(text(), "MediaWiki")]' ) then
    return string.match( atom.xml:xpath( '//generator/text()' ), "^MediaWiki (.*)$" )
  end

end

Sedusa.hints["WordPress"] = {
  '//meta[@name="generator" and starts-with(@content,"WordPress")]',
  '//head/link[@rel="stylesheet" and @type="text/css" and contains( @href, "/wp-content/")]',
  '//div[@id="content"]/div[@class="post" and starts-with(@id, "post-") and div[@class="posttitle"] and 
div[@class="postmeta"] and div[@class="postbody"] and div[@class="postfooter"]]',
}

Sedusa.verify["WordPress"] = function( u )
  local document = Sedusa.get_document( u )

  -- look in generator meta tag
  local generator = document.xml:xpath( '//meta[@name="generator"]/@content' )
  if generator and string.match( generator, "WordPress (.*)" ) then
    return string.match( generator, "WordPress (.*)" )
  end

  -- look in atom feed
  local atom = document.xml:xpath( '//link[@rel="alternate" and @type="application/atom+xml"]/@href' ) 
  local feed = Sedusa.get_document( atom )
  if feed.xml:xpath( '//generator[text()="WordPress"]/@version' ) then
    return feed.xml:xpath( '//generator[text()="WordPress"]/@version' )
  end

  return "Version not identified."
end

Sedusa.hints["Serendipity"] = {
  '//meta[@name="Powered-By" and starts-with(@content, "Serendipity")]'
}

Sedusa.verify["Serendipity"] = function( u ) 
  local document = Sedusa.get_document( u )

  local generator = document.xml:xpath( '//meta[@name="Powered-By"]/@content' )
  if generator and string.match( generator, "Serendipity v[.](.*)" ) then
    return string.match( generator, "Serendipity v[.](.*)" )
  end

end

Sedusa.hints["Trac"] = {
  '//div[@id="footer"]/a[@id="tracpowered" and @href="http://trac.edgewall.org/"]/img[@alt="Trac Powered"]',
}

Sedusa.verify["Trac"] = function( u ) 
  local document = Sedusa.get_document( u )

  local version = document.xml:xpath( '//div[@id="footer" and a[@id="tracpowered" and 
@href="http://trac.edgewall.org/"]]/p[@class="left"]/a/strong/text()' )
  if version and version:match( "Trac (.*)" ) then
    return version:match( "Trac (.*)" )
  end

end

Attachment: signature.asc
Description: OpenPGP digital signature

_______________________________________________
Sent through the nmap-dev mailing list
http://cgi.insecure.org/mailman/listinfo/nmap-dev
Archived at http://SecLists.Org