From c5fd097b79d4e9729c46cb9558c91da2a955e705 Mon Sep 17 00:00:00 2001 From: Wouter R Date: Fri, 20 Sep 2013 23:38:20 +0200 Subject: [PATCH] Replace URL decoder with a simpler implementation. --- extra/api-stresstest.sh | 11 ++-- src/main.lua | 2 +- src/rest/request.lua | 66 ++++++++++---------- src/test/test_urlcode.lua | 70 +++++++++++++++++++++ src/util/urlcode.lua | 124 ++++++++++++++++++++++++-------------- 5 files changed, 188 insertions(+), 85 deletions(-) create mode 100644 src/test/test_urlcode.lua diff --git a/extra/api-stresstest.sh b/extra/api-stresstest.sh index 6a7a679..1e4797e 100755 --- a/extra/api-stresstest.sh +++ b/extra/api-stresstest.sh @@ -14,12 +14,13 @@ WIFIBOX_IP=192.168.5.1 #WIFIBOX_IP=192.168.10.1 -API_BASE=$WIFIBOX_IP/d3dapi +#API_BASE=$WIFIBOX_IP/d3dapi +API_BASE=$WIFIBOX_IP/cgi-bin/d3dapi WGET=wget #REQUEST_PATH=network/status REQUEST_PATH=printer/print #POST_PARMS=--post-data=xyzzy -POST_PARMS=--post-file=lorem.txt +POST_PARMS=--post-file=200k.gcode RETRIES=1 @@ -30,14 +31,14 @@ while true; do $WGET -q -O - $POST_PARMS -t $RETRIES $API_BASE/$REQUEST_PATH 2>&1 >/dev/null #check $? (and time spent?) #print line every 100 counts or when a timeout/error occurs? - + if [ $? -gt 0 ]; then echo "response error at counter: $counter" fi - + if [ `expr $counter % 25` -eq 0 ]; then echo "counter: $counter" fi - + counter=`expr $counter + 1` done diff --git a/src/main.lua b/src/main.lua index d60c244..709cc1b 100644 --- a/src/main.lua +++ b/src/main.lua @@ -160,7 +160,7 @@ local function init(environment) return true end - local function main(environment) +local function main(environment) local rq = RequestClass.new(environment, postData, confDefaults.DEBUG_API) if rq:getRequestMethod() == 'CMDLINE' and rq:get('autowifi') ~= nil then diff --git a/src/rest/request.lua b/src/rest/request.lua index d569284..3e86f4c 100644 --- a/src/rest/request.lua +++ b/src/rest/request.lua @@ -23,7 +23,7 @@ M.resolutionError = nil --non-nil means function could not be resolved local function kvTableFromUrlEncodedString(encodedText) local args = {} if (encodedText ~= nil) then - urlcode.parsequery(encodedText, args) + urlcode.parsequeryNoRegex(encodedText, args) end return args @@ -31,9 +31,9 @@ end local function kvTableFromArray(argArray) local args = {} - + if not argArray then return args end - + for _, v in ipairs(argArray) do local split = v:find("=") if split ~= nil then @@ -42,7 +42,7 @@ local function kvTableFromArray(argArray) args[v] = true end end - + return args end @@ -65,18 +65,18 @@ end local function resolveApiModule(modname) if modname == nil then return nil, "missing module name" end if string.find(modname, '_') == 1 then return nil, "module names starting with '_' are preserved for internal use" end - + local reqModName = 'rest.api.api_' .. modname local ok, modObj - + if confDefaults.DEBUG_PCALLS then ok, modObj = true, require(reqModName) else ok, modObj = pcall(require, reqModName) end - + if ok == false then return nil, "API module does not exist" end if modObj == nil then return nil, "API module could not be found" end if modObj.isApi ~= true then return nil, "module is not part of the CGI API" end - + return modObj end @@ -95,43 +95,43 @@ end -- @see resolveApiModule local function resolveApiFunction(modname, funcname, requestMethod) local resultData = {} - + if funcname and string.find(funcname, "_") == 1 then return nil, "function names starting with '_' are preserved for internal use" end - + local mod, msg = resolveApiModule(modname) - + if mod == nil then -- error is indicated by leaving out 'func' key and adding 'notfound'=true resultData.notfound = true resultData.msg = msg return resultData end - + if (funcname == nil or funcname == '') then funcname = GLOBAL_API_FUNCTION_NAME end --treat empty function name as nil local rqType = requestMethod == 'POST' and 'POST' or 'GET' local fGeneric = mod[funcname] local fWithMethod = mod[funcname .. '_' .. rqType] local funcNumber = tonumber(funcname) - + if (type(fWithMethod) == 'function') then resultData.func = fWithMethod resultData.accessType = rqType - + elseif (type(fGeneric) == 'function') then resultData.func = fGeneric resultData.accessType = 'ANY' - + elseif funcNumber ~= nil then resultData.func = mod[GLOBAL_API_FUNCTION_NAME .. '_' .. rqType] resultData.accessType = rqType - + if not resultData.func then resultData.func = mod[GLOBAL_API_FUNCTION_NAME] resultData.accessType = 'ANY' end - + resultData.blankArg = funcNumber - + else local otherRqType = rqType == 'POST' and 'GET' or 'POST' local fWithOtherMethod = mod[funcname .. '_' .. otherRqType] @@ -143,7 +143,7 @@ local function resolveApiFunction(modname, funcname, requestMethod) resultData.notfound = true end end - + return resultData end @@ -158,7 +158,7 @@ setmetatable(M, { --NOTE: if debugging is enabled, commandline arguments 'm' and 'f' override requested module and function function M.new(environment, postData, debugEnabled) local self = setmetatable({}, M) - + --NOTE: is it correct to assume that absence of REQUEST_METHOD indicates command line invocation? self.requestMethod = environment['REQUEST_METHOD'] if type(self.requestMethod) == 'string' and self.requestMethod:len() > 0 then @@ -168,16 +168,16 @@ function M.new(environment, postData, debugEnabled) else self.requestMethod = 'CMDLINE' end - + self.cmdLineArgs = kvTableFromArray(arg) self.getArgs = kvTableFromUrlEncodedString(environment['QUERY_STRING']) self.postArgs = kvTableFromUrlEncodedString(postData) self.pathArgs = arrayFromPath(environment['PATH_INFO']) - + -- override path arguments with command line parameter and allow to emulate GET/POST if debugging is enabled *and* if the autowifi special command wasn't mentioned if debugEnabled and self.requestMethod == 'CMDLINE' and self:get('autowifi') == nil then self.pathArgs = arrayFromPath(self.cmdLineArgs['p']) - + if self.cmdLineArgs['r'] == 'GET' or self.cmdLineArgs['r'] == nil then self.requestMethod = 'GET' self.getArgs = self.cmdLineArgs @@ -189,19 +189,19 @@ function M.new(environment, postData, debugEnabled) end end table.remove(self.pathArgs, 1) --drop the first 'empty' field caused by the opening slash of the query string - - + + if #self.pathArgs >= 1 then self.requestedApiModule = self.pathArgs[1] end if #self.pathArgs >= 2 then self.requestedApiFunction = self.pathArgs[2] end - + if self.requestedApiModule == '' then self.requestedApiModule = nil end if self.requestedApiFunction == '' then self.requestedApiFunction = nil end - - + + -- Perform module/function resolution local rData = resolveApiFunction(self:getRequestedApiModule(), self:getRequestedApiFunction(), self.requestMethod) local modFuncInfo = (self:getRequestedApiModule() or "<>") .. "/" .. (self:getRequestedApiFunction() or "<>") - + if rData.func ~= nil then --function (possibly the global one) could be resolved self.resolvedApiFunction = rData.func if rData.blankArg ~= nil then --apparently it was the global one, and we received a 'blank argument' @@ -220,7 +220,7 @@ function M.new(environment, postData, debugEnabled) else self.resolutionError = "module/function '" .. modFuncInfo .. "' can only be accessed with the " .. rData.accessType .. " method" end - + return self end @@ -266,14 +266,14 @@ end function M:handle() local modname = self:getRequestedApiModule() local resp = ResponseClass.new(self) - + if (self.resolvedApiFunction ~= nil) then --we found a function (possible the global function) --invoke the function local ok, r if confDefaults.DEBUG_PCALLS then ok, r = true, self.resolvedApiFunction(self, resp) else ok, r = pcall(self.resolvedApiFunction, self, resp) end - + --handle the result if ok == true then return resp, nil @@ -285,7 +285,7 @@ function M:handle() resp:setError("cannot call function or module '" .. (modname or "") .. "/" .. (self:getRequestedApiFunction() or "") .. "' ('" .. self.resolutionError .. "')") return resp, ("cannot call requested API function ('" .. self.resolutionError .. "')") end - + return resp end diff --git a/src/test/test_urlcode.lua b/src/test/test_urlcode.lua new file mode 100644 index 0000000..f9d0072 --- /dev/null +++ b/src/test/test_urlcode.lua @@ -0,0 +1,70 @@ +-- TODO: also test malformed query strings +local urlcode = require("util.urlcode") + +local M = { + _is_test = true, + _skip = {}, + _wifibox_only = {} +} + +-- NOTE: the previous approach using #t1 and #t2 was too naive and only worked for tables with contiguous ranges of numeric keys. +local function compareTables(t1, t2) + local len = 0 + + for k1,v1 in pairs(t1) do + len = len + 1 + if t2[k1] ~= v1 then return false end + end + + for _ in pairs(t2) do len = len - 1 end + + return len == 0 and true or false +end + +local queryTexts = { + [1] = "k1=v1&k2=v2x&k3yy=v3", + [2] = "k1=v1&k2=v2x&k3yy=v3&", + [3] = "k1=v1&k2=v2x&k3yy=v3&=", + [4] = "k1=v1&k2=v2x&k3yy=v3& =", + [5] = "" +} + +local queryTables = { + [1] = { ["k1"] = "v1", ["k2"] = "v2x", ["k3yy"] = "v3" }, + [2] = { ["k1"] = "v1", ["k2"] = "v2x", ["k3yy"] = "v3" }, + [3] = { ["k1"] = "v1", ["k2"] = "v2x", ["k3yy"] = "v3" }, + [4] = { ["k1"] = "v1", ["k2"] = "v2x", ["k3yy"] = "v3", [" "] = "" }, + [5] = {} +} + +function M:_setup() + local longValue = "" + for i=1,5000 do + longValue = longValue .. i .. ": abcdefghijklmnopqrstuvwxyz\n" + end + + table.insert(queryTexts, "shortkey=&longkey=" .. longValue) + table.insert(queryTables, { ["shortkey"] = "", ["longkey"] = longValue }) +end + +function M:_teardown() +end + + +function M:test_parsequery() + for i=1,#queryTexts do + local args = {} + urlcode.parsequery(queryTexts[i], args) + assert(compareTables(queryTables[i], args)) + end +end + +function M:test_parsequeryNoRegex() + for i=1,#queryTexts do + local args = {} + urlcode.parsequeryNoRegex(queryTexts[i], args) + assert(compareTables(queryTables[i], args)) + end +end + +return M diff --git a/src/util/urlcode.lua b/src/util/urlcode.lua index 2edf4cd..8a1f373 100644 --- a/src/util/urlcode.lua +++ b/src/util/urlcode.lua @@ -15,32 +15,32 @@ local _M = {} -- Converts an hexadecimal code in the form %XX to a character local function hexcode2char (h) -return strchar(tonumber(h,16)) + return strchar(tonumber(h,16)) end ---------------------------------------------------------------------------- -- Decode an URL-encoded string (see RFC 2396) ---------------------------------------------------------------------------- function _M.unescape (str) -str = gsub (str, "+", " ") -str = gsub (str, "%%(%x%x)", hexcode2char) -str = gsub (str, "\r\n", "\n") -return str + str = gsub (str, "+", " ") + str = gsub (str, "%%(%x%x)", hexcode2char) + str = gsub (str, "\r\n", "\n") + return str end -- Converts a character to an hexadecimal code in the form %XX local function char2hexcode (c) -return strformat ("%%%02X", strbyte(c)) + return strformat ("%%%02X", strbyte(c)) end ---------------------------------------------------------------------------- -- URL-encode a string (see RFC 2396) ---------------------------------------------------------------------------- function _M.escape (str) -str = gsub (str, "\n", "\r\n") -str = gsub (str, "([^0-9a-zA-Z ])", char2hexcode) -- locale independent -str = gsub (str, " ", "+") -return str + str = gsub (str, "\n", "\r\n") + str = gsub (str, "([^0-9a-zA-Z ])", char2hexcode) -- locale independent + str = gsub (str, " ", "+") + return str end ---------------------------------------------------------------------------- @@ -52,21 +52,21 @@ end -- (in the order they came). ---------------------------------------------------------------------------- function _M.insertfield (args, name, value) -if not args[name] then -args[name] = value -else -local t = type (args[name]) -if t == "string" then -args[name] = { -args[name], -value, -} -elseif t == "table" then -tinsert (args[name], value) -else -error ("CGILua fatal error (invalid args table)!") -end -end + if not args[name] then + args[name] = value + else + local t = type (args[name]) + if t == "string" then + args[name] = { + args[name], + value, + } + elseif t == "table" then + tinsert (args[name], value) + else + error ("CGILua fatal error (invalid args table)!") + end + end end ---------------------------------------------------------------------------- @@ -78,13 +78,45 @@ end -- @param args Table where to store the pairs. ---------------------------------------------------------------------------- function _M.parsequery (query, args) -if type(query) == "string" then -local insertfield, unescape = _M.insertfield, _M.unescape -gsub (query, "([^&=]+)=([^&=]*)&?", -function (key, val) -_M.insertfield (args, unescape(key), unescape(val)) -end) + if type(query) == "string" then + local insertfield, unescape = _M.insertfield, _M.unescape + gsub (query, "([^&=]+)=([^&=]*)&?", + function (key, val) + _M.insertfield (args, unescape(key), unescape(val)) + end) + end end + +---------------------------------------------------------------------------- +-- Parse url-encoded request data without using regular expressions +-- (the query part of the script URL or url-encoded post data) +-- +-- Each decoded (name=value) pair is inserted into table [[args]] +-- @param query String to be parsed. +-- @param args Table where to store the pairs. +---------------------------------------------------------------------------- +function _M.parsequeryNoRegex (query, args) + if type(query) == "string" then + local insertfield, unescape = _M.insertfield, _M.unescape + + local k = 1 + while true do + local v = query:find('=', k+1, true) -- look for '=', assuming a key of at least 1 character and do not perform pattern matching + if not v then break end -- no k/v pairs left + + v = v + 1 + local ampersand = query:find('&', v, true) + if not ampersand then ampersand = 0 end -- 0 will become -1 in the substring call below...meaning end of string + + local key = query:sub(k, v-1) + local value = query:sub(v, ampersand - 1) + insertfield (args, unescape(key), unescape(value)) + + if ampersand == 0 then break end -- we couldn't find any ampersands anymore so this was the last k/v + + k = ampersand + 1 + end + end end ---------------------------------------------------------------------------- @@ -94,21 +126,21 @@ end -- @return String with the resulting encoding. ---------------------------------------------------------------------------- function _M.encodetable (args) -if args == nil or next(args) == nil then -- no args or empty args? -return "" -end -local escape = _M.escape -local strp = "" -for key, vals in pairs(args) do -if type(vals) ~= "table" then -vals = {vals} -end -for i,val in ipairs(vals) do -strp = strp.."&"..escape(key).."="..escape(val) -end -end --- remove first & -return strsub(strp,2) + if args == nil or next(args) == nil then -- no args or empty args? + return "" + end + local escape = _M.escape + local strp = "" + for key, vals in pairs(args) do + if type(vals) ~= "table" then + vals = {vals} + end + for i,val in ipairs(vals) do + strp = strp.."&"..escape(key).."="..escape(val) + end + end + -- remove first & + return strsub(strp,2) end return _M